diff --git a/docs/cultcache-union-parity-map.md b/docs/cultcache-union-parity-map.md index 1442abdc..ecf03f47 100644 --- a/docs/cultcache-union-parity-map.md +++ b/docs/cultcache-union-parity-map.md @@ -260,7 +260,7 @@ landed in `ead5260c`, Python in `5618d8dd`, Rust in `c028c511`. TypeScript is no |---|---|---| | C# | `CultPayloadDepth.Check` (`src/GameCult.Caching.MessagePack/CultPayloadDepth.cs`), limit `CultMessagePackSecurity.MaximumPayloadDepth`. MessagePack's own graph counter is set to the same value as a backstop only. | Write: `CultDocumentMessagePackSerialization.SerializeUntyped`. Read: `DeserializeUntyped`. The cache store (`SerializePayload`/`DeserializePayload`), the directory store, `CultMesh` and single-file documents all call those two; C# has no separate raw-envelope or prepared-batch door. | | Rust | `check_record_payload` (private; depth, then no bytes after the first value) over `first_value_end`, and `MAX_PAYLOAD_DEPTH`; `cultcache_rs::check_payload_depth` is the depth-only probe for outside callers (`packages/cultcache-rs/src/lib.rs`). | Read: `decode_payload` (get, list, `validate_envelope`/`put_envelope`, `finish_prepared_entry`). Write: `put`/`prepare_entry`/`prepare_entry_named` reach it through `finish_prepared_entry`, which decodes what it just encoded. Raw envelope: `put_raw_envelope`, the door CultMesh network puts use (`cultmesh-rs` `lib.rs`, two call sites). Prepared batch: `put_prepared_batch` via `require_record_payload`. Load: `pull_all_backing_stores` and `load_soa` run `require_record_payload` on every record, so a store holding a payload past the bound (or with bytes after it) is refused at load, and the cache then refuses every write and removal (`CultCacheLoadRefused`) until a load succeeds. cultnet-rs: every free-form or skippable wire payload decodes through `cultcache_rs::decode_payload` (message decode in `contracts.rs`, `provider_session`, `media_stream_wire`, `replication`, `service_identity`). | -| Python | `_refuse_past_depth_limit` and `MAX_PAYLOAD_DEPTH` (`packages/cultcache-py/src/cultcache_py/documents.py`); a msgpack `StackError` on read maps to the same error. | `DocumentDefinition.encode_payload` (write) and `decode_payload` (read). `CultCache.put`, `put_envelope`, `put_envelopes` and load-time hydration go through them; there is no raw-envelope door. cultmesh-py round trips call the same pair. | +| Python | `_containers` (the one depth walk, through `_check_payload_tree`) and `MAX_PAYLOAD_DEPTH` (`packages/cultcache-py/src/cultcache_py/documents.py`); a msgpack `StackError` on read maps to the same error. | `DocumentDefinition.encode_payload` (write) and `decode_payload` (read). `CultCache.put`, `put_envelope`, `put_envelopes` and load-time hydration go through them; there is no raw-envelope door. cultmesh-py round trips call the same pair. | - **Doors deliberately not routed.** cultnet-rs `idunn_lifecycle_brake`, `provider_health_contracts` and `runtime_authority_contracts` decode records whose positional array length is checked exactly before decoding. A diff --git a/docs/research/py-msgpack-default/probe2.py b/docs/research/py-msgpack-default/probe2.py index 259320eb..d9c87793 100644 --- a/docs/research/py-msgpack-default/probe2.py +++ b/docs/research/py-msgpack-default/probe2.py @@ -14,7 +14,7 @@ raw = msgpack.unpackb(open(path, "rb").read(), raw=False) print("store header", raw[0], "catalog entry", raw[1][0][:5] if raw[1] else None) print("record", raw[2][0][:3], "payload", raw[2][0][3][:40]) # MessagePack-default reader, today's _messagepack pair standing in for the proposed default -new = cc.define_document_type("brokkr.sync.var.v0", payload_encoder=lambda t: msgpack.packb(t, use_bin_type=True), payload_decoder=lambda b: msgpack.unpackb(b, raw=False)) +new = cc.define_document_type("brokkr.sync.var.v0") # historical: the probe stood a msgpack pair in for the proposed default, now the default try: open_cache(new); print("new reader: ACCEPTED") except Exception as e: print("new reader refused:", type(e).__module__, type(e).__name__, str(e)[:80]) # opt-in JSON pair reads it diff --git a/docs/research/py-typed-leaves-probe/write_probe_stores.py b/docs/research/py-typed-leaves-probe/write_probe_stores.py index bca857d3..371df5e8 100644 --- a/docs/research/py-typed-leaves-probe/write_probe_stores.py +++ b/docs/research/py-typed-leaves-probe/write_probe_stores.py @@ -3,7 +3,7 @@ import sys, os sys.path.insert(0, r"F:\Projects\CultLib\packages\cultcache-py\src"); sys.path.insert(0, os.path.dirname(__file__)) import proto_leaves as L from cultcache_py import CultCache, SingleFileMessagePackBackingStore -from cultcache_py.documents import define_document_type +from cultcache_py.documents import PayloadCodec, define_document_type from cultcache_py.backing_store import CultCacheSchemaCatalogMember as M NS = "GameCult.Caching.Tests.PyLeavesProbe" @@ -25,7 +25,7 @@ def payload(doc_id): def write(path, members, tree_for): doc = define_document_type( "cultcache.py-leaves-probe", encode=lambda v: v, decode=lambda v: v, - payload_encoder=L.write, payload_decoder=lambda b: b, name=lambda v: v[0], + payload_codec=PayloadCodec("leaves-probe", L.write, lambda b: b), name=lambda v: v[0], schema_name="cultcache.py-leaves-probe", schema_version="cultcache.py_leaves_probe.v1", members=[M(slot=s, member_name=n, type_name=t, is_name=(s == 0)) for s, n, t in members]) if os.path.exists(path): os.remove(path) diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index 30b60850..29544715 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -4,12 +4,13 @@ ### Changed +- Breaking: `define_document_type` writes and reads MessagePack payloads by default (`cultlib-gaps:ruling:py-msgpack-default`); it wrote JSON until 0.3. A definition's `payload_codec` (`PayloadCodec`, with `MESSAGEPACK_PAYLOAD_CODEC` and `JSON_PAYLOAD_CODEC` exported) owns its payload format, replacing the `payload_encoder` and `payload_decoder` parameters and fields, so no definition carries a second author of the format. A store written by 0.3 or earlier holds JSON: define its types with `payload_codec=JSON_PAYLOAD_CODEC` to read them, or write a new MessagePack store with `convert(source, destination, definitions)`, which never writes its source. The MessagePack codec takes the keys, integers, timestamps and nesting the C# reference's object reader takes and refuses the rest on read and on write (`CultCachePayloadFormatError`, `PayloadCodecFault`; see the README, Payload format); the single-file store's reader takes the same map keys. A 0.3 JSON payload that is an object or list is refused by count as a payload with bytes after its first MessagePack value, a `ValueError`. `define_database_entry_type` inherits the checked codec too: main wrote `{b'k': 1}` in an untyped slot, which is now refused on write, and a bin map key in an entry payload is refused on read where main read it, so a main-era store with a bin-keyed entry fails to load. The checked codec walks each tree twice (the depth walk, then the carried-value check), which roughly doubles entry and document codec cost; the double walk is left in place. The JSON codec's `depth_faults=(RecursionError,)` gives a JSON payload nested past the interpreter's stack the typed `CultCachePayloadDepthError`, which resolves the deferred `cut-union-bounds-py.s2.depth-typed-refusal-lost` for the JSON codec only. This is the same minor bump as the rest of this section: 0.4.0 is not yet tagged. - `parse_stored_at` (was `_parse_stored_at`) is public: cultmesh-py judges a peer's `storedAt` by it. A peer's `document_put_raw` whose `storedAt` is missing, empty or not a stamp is refused `message_invalid` (field `document`) and nothing is minted or stored (variants:ruling:raw-put-without-stamp-refused). A subscribed server's stampless put now raises `ValueError` from `CultMeshClient.read_next_change` instead of being stored under a minted stamp. ### Added - `CultCache.update` and `update_global` return `UpdateResult(outcome, document)`; `UpdateOutcome` is `UPDATED`, `DECLINED` or `LOST_RACE`. The single-file store gains `read(key)` (the record and its content version, the SHA-256 C# computes) and `commit_if(envelope, expected)`, the version is the SHA-256 of the record re-encoded as `[key, schemaId, storedAt, payload]` with minimal MessagePack encodings, which is the SHA-256 of the record's stored bytes exactly when the writer encoded it minimally (every writer in this library does; a record another writer encoded with a wider-than-needed length or integer has the version of its minimal form, not of its bytes). Both looked up by the cache and not `BackingStore` protocol members (variants:cut_spec:cut-update-with-retry-py). -- `read_payload_tree(type_name, payload, decoder=None)`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged. +- `read_payload_tree(type_name, payload, decoder=None, depth_faults=(msgpack.exceptions.StackError,))`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged. ### Changed @@ -28,7 +29,7 @@ - `push_all(envelopes, soft=False)` on the single-file and JSONL stores and on the `BackingStore` protocol takes the soft flag, as TypeScript, Rust and C# do: a soft push writes only where nothing is at the path; over a store the reader reads, empty included, it returns without writing, and over a file the reader cannot read it refuses (`StoreUnreadableError` for the single-file store). The `BackingStore` protocol has `pull_or_absent()`: the envelopes the reader reads, `None` when nothing is at the path, a raise when the path cannot be read; its default is `pull_all()`, which cannot say absent, so a store that inherits it is always read. The default `push_all` decides soft by it (writes where it says absent, skips over a store it read, refuses where it raises), and the single-file and JSONL stores expose their pull through it; an inheriting store that can tell nothing-at-the-path overrides `pull_or_absent`, not `push_all`. Before, a soft push through the default always skipped. A store kind that overrides `push_all(envelopes)` without the keyword fails when a caller passes `soft`. - A write at a store path that is a link, live or dangling, is refused with the new `StoreLinkedError` (an `OSError`, exported from `cultcache_py`; `.path` names the path) before the lock and the replace, for the single-file store and for `JsonLinesBackingStore` (`push`, `push_all`, `delete`), and nothing is written. Before, a write at a live link replaced the link with a regular file, and at a dangling link both stores raised `FileNotFoundError`. A caller catching `FileNotFoundError` there now catches `StoreLinkedError` (`OSError`); configure the link's target as the store path. On Windows a junction or any reparse point counts as a link. Reads through a live link work. - A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`. -- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` ` payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#. +- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` ` payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on a codec go through it; a codec whose `decode` raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#. - A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it. - Added `foreign_records()` and the frozen `ForeignRecord(key, schema_id, schema_name, stored_at)`, sorted by key in UTF-16 code-unit order. `stores.retyped` and `stores.written_owns` lose their underscore: the cache judge and the single-file store's commit guard ask one predicate. diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 0a03a671..c06cc2a9 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -31,7 +31,7 @@ discovery, sessions, local server) build on it, mirroring `cultcache-rs` / second store, or a second generic store, raises `CultCacheError`; a write touches exactly one store and nothing is mirrored; once any store is attached, a write whose type has no home raises -- the JSONL store uses only the Python standard library for bootstrap consumers +- `define_document_type(...)` payloads are MessagePack by default; `JSON_PAYLOAD_CODEC` reads stores written by 0.3 and earlier - `define_database_entry_type(...)` emits Rust/C#-style slot-indexed MessagePack array payloads for cross-runtime `DatabaseEntry` contracts @@ -131,7 +131,7 @@ reclassified; a unique key declared a role answers no lookups and stops refusing Backing stores persist envelopes, not caller domain objects. The cache decodes payloads through registered document definitions and rejects unknown type discriminators. This keeps Python's dynamic runtime from turning persistence into an open polymorphic sewer with a cheerful docstring. -`JsonLinesBackingStore` is the dependency-free control-plane store. It rewrites an atomic JSONL snapshot and base64-encodes payload bytes. It is designed for compact state spines, settings, ledgers, and bootstrap surfaces. +`JsonLinesBackingStore` is the control-plane store. It rewrites an atomic JSONL snapshot and base64-encodes the payload bytes its definition's codec wrote. It is designed for compact state spines, settings, ledgers, and bootstrap surfaces. `SingleFileMessagePackBackingStore` follows the shared CultCache v1 store shape used by the TypeScript, Rust, and C# runtimes: @@ -150,10 +150,55 @@ Large corpora should use a sharded store or a real database. A single snapshot f A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-py from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) +## Payload format + +A definition's `payload_codec` owns its payload bytes. The default, +`MESSAGEPACK_PAYLOAD_CODEC`, writes MessagePack; a store holds no format marker, +so the registered definition decides how every record is read. A record in another +format is refused when it is not one value of the definition's codec: a JSON object, +list, string, `true` or `null` read as MessagePack is refused by count (` +payload has N bytes after its first MessagePack value`, a `ValueError`), and a payload +the codec cannot read or carry is a `CultCachePayloadFormatError` or +`CultCachePayloadDepthError`. The one exception is a JSON document that is a single +digit (`0` to `9`), below. + +- Map keys are `str`, integer, float, `bool` or `msgpack.Timestamp`, as the C# + reference's object reader takes them; `None`, `bytes`, list and map keys are + refused, on read and on write. Two keys Python holds as one are refused on read + as `duplicate_map_key`, which includes `1` beside `1.0` or `True`: the reference + keeps those apart and a Python `dict` cannot. The same keys are refused on + write as `duplicate_map_key`, which includes a `Float32` key beside an equal + int, bool or float key, and two NaN keys, which are one key: the codec never + writes bytes its own reader refuses. One pair of keys is the exception to the reference, see Wire Parity. +- Arrays and maps nest at most 256 deep (`MAX_PAYLOAD_DEPTH`), on read and on + write, in the codec and in the definition alike. +- Integers beyond 64 bits (outside `-(2**63)..2**64-1`) are refused on write as + `integer_out_of_range`; MessagePack has no wider integer. +- Timestamps lie within years 1 to 9999 (`.NET` `DateTime`), and no extension type + but the timestamp exists: a `msgpack.ExtType` value or a timestamp outside that + range is refused on read and on write as `value_kind` or `timestamp_out_of_range`. +- A store written by cultcache-py 0.3 or earlier holds JSON payloads. Define each + type with `payload_codec=JSON_PAYLOAD_CODEC` to read it, or write it as a new store + with `convert(source, destination, definitions)`. Stop every writer of the source + first. `destination` must not exist; every record of the listed definitions is + rewritten under the definitions' own codec, records of other types keep their + bytes, and the new file is fsynced before the call returns. Open it with the + default definitions, then swap it in yourself. The source is never written. A + refused record raises `CultCacheError` naming the type, record key and code, and + leaves no destination; a killed run leaves the source as it was and a partial + destination, so delete the destination and run again. +- A JSON payload that is a single digit (`0` to `9`) is also a valid one-byte + MessagePack integer and reads as 48 to 57 under the default, with no refusal. Only a + document whose whole value is such a digit meets this; objects, lists and + longer numbers never do. +- A `PayloadCodec` you build owns the nesting depth of nodes the library's walk + cannot see (anything that is not a mapping, sequence or set), and may raise + `PayloadCodecFault(code, kind)` for a key or integer its format cannot carry. + ## DatabaseEntry Slot Contract -For cross-runtime cache entries, use `define_database_entry_type(...)` instead -of the generic JSON formatter: +For cross-runtime cache entries, use `define_database_entry_type(...)`, which +fixes the MessagePack slot-array payload: ```python from dataclasses import dataclass @@ -248,6 +293,10 @@ carries the value. ## Wire Parity +Bytes match the C# reference for typed slots. An untyped (`object`) slot differs in +what it writes: C# writes a boxed integer 1 as `d2 00000001`, this runtime writes the +fixint `01`; each runtime reads the other's bytes. + See [docs/python-runtime-parity.md](../../docs/python-runtime-parity.md) for the evidence map across the three Python packages. @@ -258,6 +307,12 @@ python -m cultcache_py.interop write --file cache.cc --runtime-id python python -m cultcache_py.interop read --file cache.cc ``` +One deliberate exception to leaf acceptance following the reference: Python reads `float32` and `float64` map keys as +one float, so a map holding a `float32` NaN key beside a `float64` NaN key is refused on read and on write as +`duplicate_map_key`. C# reads that map as two keys, from the bytes `82 ca ffc00000 01 cb fff8000000000000 02` (fixint +values) and from the bytes it writes itself, `82 ca ffc00000 d2 00000001 cb fff8000000000000 d2 00000002` (`float.NaN` +and `double.NaN` keys with values 1 and 2, written as int32); every pair of NaN keys of one width is refused by both runtimes. + `packages/cultcache-ts/test/cult-cache.test.ts` includes Python in the shared CultCache v1 parity matrix with TypeScript, Rust, and C#. `cultcache_py` ships a `py.typed` marker so downstream type checkers can inspect the package surface. diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index 0c26cb25..363c338e 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -1,13 +1,18 @@ from .backing_store import BackingStore, CultCacheEnvelope from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember from . import leaves -from .cache import CultCache, CultCacheBuilder, ForeignRecord, UpdateOutcome, UpdateResult +from .cache import CultCache, CultCacheBuilder, ForeignRecord, UpdateOutcome, UpdateResult, convert from .documents import ( MAX_PAYLOAD_DEPTH, + JSON_PAYLOAD_CODEC, MAX_SLOT_KEY, + MESSAGEPACK_PAYLOAD_CODEC, CultCachePayloadDepthError, + CultCachePayloadFormatError, DatabaseEntryField, DocumentDefinition, + PayloadCodec, + PayloadCodecFault, UnionArm, UnreadSlots, database_entry_field, @@ -26,15 +31,20 @@ __all__ = [ "CultCache", "CultCacheBuilder", "CultCachePayloadDepthError", + "CultCachePayloadFormatError", "CultCacheEnvelope", "CultCacheSchemaCatalogEntry", "CultCacheSchemaCatalogMember", "DatabaseEntryField", "DocumentDefinition", "ForeignRecord", + "JSON_PAYLOAD_CODEC", "JsonLinesBackingStore", "MAX_PAYLOAD_DEPTH", "MAX_SLOT_KEY", + "MESSAGEPACK_PAYLOAD_CODEC", + "PayloadCodec", + "PayloadCodecFault", "SchemaConflictError", "SingleFileMessagePackBackingStore", "SlotCodec", @@ -52,4 +62,5 @@ __all__ = [ "define_document_type", "define_record_codec", "define_union", + "convert", ] diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index f82bd7d5..dbb09997 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -3,14 +3,24 @@ import enum import functools import math +import os import threading from decimal import Decimal from dataclasses import dataclass, field, replace +from pathlib import Path from typing import Any, Callable, Generic, Iterable, TypeVar from .backing_store import BackingStore, CultCacheEnvelope -from .documents import DocumentDefinition, UnreadSlots, extract_value -from .stores import SchemaConflictError, parse_stored_at, retyped, stored_at_over, written_owns +from .documents import JSON_PAYLOAD_CODEC, DocumentDefinition, PayloadCodec, UnreadSlots, extract_value +from .stores import ( + JsonLinesBackingStore, + SchemaConflictError, + SingleFileMessagePackBackingStore, + parse_stored_at, + retyped, + stored_at_over, + written_owns, +) T = TypeVar("T") F = TypeVar("F", bound=Callable[..., Any]) @@ -1042,3 +1052,74 @@ class CultCache: if document is not None: return document return self._state.documents_by_schema_name.get(envelope.type) + + +def convert( + source: SingleFileMessagePackBackingStore | JsonLinesBackingStore, + destination: str | os.PathLike[str], + definitions: Iterable[DocumentDefinition[Any]], + *, + source_codec: PayloadCodec = JSON_PAYLOAD_CODEC, +) -> int: + """Write the records of `source` to a new store of the same kind at `destination`, the records of `definitions` + rewritten from `source_codec` to each definition's own codec, and return the count rewritten. Records of other types + keep their bytes. Stop every writer of the source before calling. The destination keeps the header the source was read under (a `cultcache.store.v3` source stays v3). `destination` must not exist: the call creates it + exclusively, and it is complete only once the call returns. A refusal (a record, a failed write) or an interrupt + after the create removes it and leaves the source as it was. A killed run leaves the source intact and a partial + destination: delete the destination and run again. A file another process creates at the destination between the + existence check and the create is refused and left alone. The source is only read, never written; swap the new + store in yourself once it opens. A count of 0 still writes the destination. + """ + if not isinstance(source, (SingleFileMessagePackBackingStore, JsonLinesBackingStore)): + raise TypeError(f"convert takes a library file store, not {type(source).__name__}") + if not source.path.exists(): + raise CultCacheError("convert: argument source does not exist (source_missing); there is nothing to convert.") + by_type: dict[str, DocumentDefinition[Any]] = {} + for definition in definitions: + by_type[definition.type] = definition + by_type.setdefault(definition.catalog_entry().schema_name, definition) + destination = Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + exists = CultCacheError( + "convert: argument destination already exists (destination_exists); convert writes only a new store, " + "and the source is unchanged." + ) + if destination.exists(): + raise exists + target = None + failure: CultCacheError | None = None + try: + try: + target = open(destination, "xb") + except FileExistsError: + pass + else: + with target: + converted: list[CultCacheEnvelope] = [] + count = 0 + for envelope in source.pull_all(): + definition = by_type.get(envelope.type) + if definition is not None: + try: + value = replace(definition, payload_codec=source_codec).decode_payload(envelope.payload) + envelope = replace(envelope, payload=definition.encode_payload(value)) + except Exception as refusal: + code = getattr(refusal, "code", type(refusal).__name__) + failure = CultCacheError( + f"Cannot convert {envelope.type} record {envelope.key}: {code}; " + "no store was written and the source is unchanged." + ) + else: + count += 1 + if failure is not None: + raise failure + converted.append(envelope) + target.write(source._file_bytes(converted)) + target.flush() + os.fsync(target.fileno()) + except BaseException: + destination.unlink(missing_ok=True) + raise + if target is None: + raise exists + return count diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index ff9039dc..ef376eff 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import struct import threading from dataclasses import MISSING, dataclass, field from collections.abc import Mapping, Sequence, Set @@ -8,7 +9,6 @@ from itertools import chain from typing import Any, Callable, Generic, Iterable, TypeVar import msgpack -from msgpack.exceptions import StackError as _MsgpackStackError from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember from .leaves import Float32, SlotCodec @@ -28,9 +28,6 @@ or skipped, the outermost one is depth 1; depth 256 is accepted and 257 refused, MAX_SLOT_KEY = 65535 """The largest slot key a record may define: the largest MessagePack array16 length.""" -_STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,) - - @dataclass(frozen=True) class UnreadSlots: """What a payload decode found past the slots this runtime declares, at any depth: the first type read with extra slots, how @@ -89,22 +86,85 @@ class CultCachePayloadDepthError(ValueError): ) -def _refuse_past_depth_limit(tree: Any, type_name: str, direction: str) -> None: - """The one nesting count. Walks a payload tree iteratively (every mapping, sequence and set that is not text or - bytes is a container), refusing the first container past `MAX_PAYLOAD_DEPTH`.""" - stack = [(tree, 1)] if _is_container(tree) else [] - while stack: - node, depth = stack.pop() - if depth > MAX_PAYLOAD_DEPTH: - raise CultCachePayloadDepthError(type_name, direction) - children = chain(node.keys(), node.values()) if isinstance(node, Mapping) else node - stack.extend((child, depth + 1) for child in children if _is_container(child)) +class PayloadCodecFault(ValueError): + """A codec raises this for a payload its format cannot carry. `code` names the rule (the library's are + `map_key_kind`, `duplicate_map_key`, `integer_out_of_range`, `depth_limit`, `value_kind` and + `timestamp_out_of_range`) and `kind` the kind of node at fault (a MessagePack + kind such as `nil` or `bin`); neither is ever a payload value. A consumer-built codec may raise it.""" + + def __init__(self, code: str, kind: str = "") -> None: + self.code = code + self.kind = kind + super().__init__(code) + + +class CultCachePayloadFormatError(ValueError): + """A payload is not one value of its definition's codec (read), or holds a key or integer the codec cannot carry. + Names the type, the codec, the direction and a `code`; `detail` is the fault's class or the kind of the key or + extension value. Never a payload byte or value.""" + + def __init__(self, type_name: str, codec: str, direction: str, code: str, detail: str) -> None: + self.type_name = type_name + self.codec = codec + self.direction = direction + self.code = code + if code == "not_one_value": + message = f"{type_name} payload is not one {codec} value ({detail}); refused on read." + elif code == "map_key_kind": + message = ( + f"{type_name} payload has a map key of kind {detail}; " + f"{codec} payloads take str, int, float, bool or timestamp keys (refused on {direction})." + ) + elif code == "duplicate_map_key": + message = f"{type_name} payload has two map keys Python holds as one; refused on {direction}." + elif code == "integer_out_of_range": + message = f"{type_name} payload has an integer beyond 64 bits; refused on write." + elif code == "value_kind": + message = ( + f"{type_name} payload holds a value of kind {detail}; " + f"{codec} payloads carry no extension type but the timestamp (refused on {direction})." + ) + elif code == "timestamp_out_of_range": + message = f"{type_name} payload holds a timestamp outside years 1 to 9999; refused on {direction}." + else: + message = f"{type_name} payload is refused by the {codec} codec ({code}); refused on {direction}." + super().__init__(message) def _is_container(node: Any) -> bool: return isinstance(node, _CONTAINERS) and not isinstance(node, _SCALAR_SEQUENCES) +def _children(node: Any) -> Iterable[Any]: + return chain(node.keys(), node.values()) if isinstance(node, Mapping) else node + + +def _containers(tree: Any) -> Iterable[Any]: + """The one depth walk over a payload tree, for every codec and every caller. Iterative, so a cycle ends at the limit: + yields every container (a mapping, sequence or set that is not text or bytes; a mapping's keys and values are both + children), the outermost at depth 1, and raises `PayloadCodecFault('depth_limit')` at the first one past + `MAX_PAYLOAD_DEPTH`.""" + stack = [(tree, 1)] if _is_container(tree) else [] + while stack: + node, depth = stack.pop() + if depth > MAX_PAYLOAD_DEPTH: + raise PayloadCodecFault("depth_limit") + yield node + stack.extend((child, depth + 1) for child in _children(node) if _is_container(child)) + + +def _check_payload_tree(tree: Any, type_name: str, direction: str) -> None: + """The definition's translation of the depth walk into its typed refusal.""" + try: + for _ in _containers(tree): + pass + except PayloadCodecFault: + refusal = CultCachePayloadDepthError(type_name, direction) + else: + return + raise refusal + + def _default_encode(value: Any) -> Any: if hasattr(value, "model_dump"): return value.model_dump() @@ -119,14 +179,6 @@ def _default_decode(value: Any) -> Any: return value -def _json_payload_encoder(value: Any) -> bytes: - return json.dumps(value, ensure_ascii=False, sort_keys=True).encode("utf-8") - - -def _json_payload_decoder(payload: bytes) -> Any: - return json.loads(payload.decode("utf-8")) - - def _pack_tree(tree: Any) -> bytes: """The bytes of the tree the leaves produced. `packb` writes everything but a `Float32`, which it refuses with a TypeError; only then does an iterative walk pack each array and map header and each value, a `Float32` through a @@ -154,27 +206,147 @@ def _pack_tree(tree: Any) -> bytes: return b"".join(out) +@dataclass(frozen=True) +class PayloadCodec: + """A codec owns one payload format: how a tree becomes bytes and back, which of its decode faults mean the payload + nests past the interpreter's own stack, and which trees it can carry: it raises `PayloadCodecFault` for a key, an + integer or a nesting its format cannot hold, and nothing it writes is refused by its own `decode`. A consumer-built codec owns + the depth of nodes the depth walk cannot see (anything that is not a Mapping, Sequence or Set).""" + + name: str + encode: Callable[[Any], bytes] + decode: Callable[[bytes], Any] + depth_faults: tuple[type[BaseException], ...] = () + + +_INTEGER_RANGE = (-(2**63), 2**64 - 1) +_TIMESTAMP_SECONDS = (-62135596800, 253402300799) +_NAN_KEY = object() +_KEY_TYPES = (str, int, float, Float32, msgpack.Timestamp) + + +def _node_kind(node: Any) -> str: + """The MessagePack kind of a node, for a refusal: a kind, never a value.""" + if node is None: + return "nil" + if isinstance(node, (bytes, bytearray, memoryview)): + return "bin" + if isinstance(node, Mapping): + return "map" + if isinstance(node, msgpack.ExtType): + return "ext" + if isinstance(node, _CONTAINERS): + return "array" + return "unsupported" + + +def _key_image(key: Any) -> Any: + """A map key as the reader holds it, and the one key identity of read and write: a `Float32` is written as float32 + and read back as that float, and every NaN is one key (the reference refuses two of them).""" + if isinstance(key, Float32): + key = struct.unpack(">f", struct.pack(">f", key.value))[0] + return _NAN_KEY if isinstance(key, float) and key != key else key + + +def _checked_map(pairs: list[tuple[Any, Any]]) -> dict[Any, Any]: + """The read hook of every MessagePack map: the keys the C# reference's object reader accepts (str, int, float, + bool, timestamp) and no duplicate.""" + for key, _ in pairs: + if not isinstance(key, _KEY_TYPES): + raise PayloadCodecFault("map_key_kind", _node_kind(key)) + if len(dict.fromkeys(_key_image(key) for key, _ in pairs)) != len(pairs): + raise PayloadCodecFault("duplicate_map_key") + return dict(pairs) + + +def _check_scalar(node: Any) -> None: + if isinstance(node, int) and not _INTEGER_RANGE[0] <= node <= _INTEGER_RANGE[1]: + raise PayloadCodecFault("integer_out_of_range") + if isinstance(node, msgpack.Timestamp) and not _TIMESTAMP_SECONDS[0] <= node.seconds <= _TIMESTAMP_SECONDS[1]: + raise PayloadCodecFault("timestamp_out_of_range") + + +def _check_carried(tree: Any) -> None: + """The one check of what a MessagePack payload carries, on read and on write: the keys, integers, timestamps and + nesting the C# reference's object reader takes, no extension type but the timestamp, and no two keys that decode + as one (a `Float32` beside an equal int, bool or float, or two NaNs), so nothing this codec writes is refused by + its own decode.""" + _check_scalar(tree) + for node in _containers(tree): + if isinstance(node, msgpack.ExtType): + raise PayloadCodecFault("value_kind", "ext") + for child in _children(node): + _check_scalar(child) + if isinstance(node, Mapping): + for key in node: + if not isinstance(key, _KEY_TYPES): + raise PayloadCodecFault("map_key_kind", _node_kind(key)) + if len(dict.fromkeys(_key_image(key) for key in node)) != len(node): + raise PayloadCodecFault("duplicate_map_key") + + +def _unpack_tree(payload: bytes) -> Any: + return msgpack.unpackb(payload, raw=False, strict_map_key=False, object_pairs_hook=_checked_map) + + +def _unpack_checked_tree(payload: bytes) -> Any: + tree = _unpack_tree(payload) + _check_carried(tree) + return tree + + +def _pack_checked_tree(tree: Any) -> bytes: + _check_carried(tree) + return _pack_tree(tree) + + +def _dump_json(tree: Any) -> bytes: + return json.dumps(tree, ensure_ascii=False, sort_keys=True).encode("utf-8") + + +def _load_json(payload: bytes) -> Any: + return json.loads(payload.decode("utf-8")) + + +MESSAGEPACK_PAYLOAD_CODEC = PayloadCodec( + "MessagePack", _pack_checked_tree, _unpack_checked_tree, depth_faults=(msgpack.exceptions.StackError,) +) +"""The default payload format. It carries what the C# reference's object reader accepts: str, int, float, bool and +timestamp keys, no duplicate key (two NaN keys are one), integers within -(2**63)..2**64-1, timestamps within years 1 to +9999, no extension type but the timestamp, and nesting no deeper than `MAX_PAYLOAD_DEPTH`. One check refuses the rest on +read and on write, so it never writes a record it cannot read.""" + +JSON_PAYLOAD_CODEC = PayloadCodec("JSON", _dump_json, _load_json, depth_faults=(RecursionError,)) +"""The 0.3 default, kept for stores written before MessagePack: it writes exactly the 0.3 bytes. Never a default.""" + + def _messagepack_payload_decoder(payload: bytes) -> Any: return msgpack.unpackb(payload, raw=False) -def read_payload_tree(type_name: str, payload: bytes, decoder: Callable[[bytes], Any] | None = None) -> Any: +def read_payload_tree( + type_name: str, + payload: bytes, + decoder: Callable[[bytes], Any] | None = None, + depth_faults: tuple[type[BaseException], ...] = (msgpack.exceptions.StackError,), +) -> Any: """The one read of a payload's tree: exactly one MessagePack value, within the nesting limit. Depth is judged first, on the value read so far, so a tail never pre-empts it (C# throws at container 257 before it reaches the tail); then a - tail is refused by count and never echoed. Raised outside the handler so no `__context__` keeps the tail.""" + tail is refused by count and never echoed. Raised outside the handler so no `__context__` keeps the tail. A + definition passes its codec's `decode` and `depth_faults`; with none this reads plain MessagePack.""" after = -1 unpacked: Any = None try: tree = (decoder or _messagepack_payload_decoder)(payload) - except _STACK_FAULTS: + except depth_faults: raise CultCachePayloadDepthError(type_name, "read") from None except msgpack.ExtraData as error: after = len(error.extra) unpacked = error.unpacked if after >= 0: - _refuse_past_depth_limit(unpacked, type_name, "read") + _check_payload_tree(unpacked, type_name, "read") raise ValueError(f"{type_name} payload has {after} bytes after its first MessagePack value (refused on read).") - _refuse_past_depth_limit(tree, type_name, "read") + _check_payload_tree(tree, type_name, "read") return tree @@ -212,6 +384,12 @@ class UnionArm: value: Any = None +def _fault_refusal(type_name: str, codec: PayloadCodec, direction: str, fault: PayloadCodecFault) -> CultCachePayloadFormatError | CultCachePayloadDepthError: + if fault.code == "depth_limit": + return CultCachePayloadDepthError(type_name, direction) + return CultCachePayloadFormatError(type_name, codec.name, direction, fault.code, fault.kind) + + @dataclass(frozen=True) class DocumentDefinition(Generic[T]): type: str @@ -226,8 +404,7 @@ class DocumentDefinition(Generic[T]): roles: dict[str, str | Extractor[T]] = field(default_factory=dict) """Selection aliases (C# `[CultRole]`): many records may share a value. Never indexed for lookup, never refuse; the member's catalog `index_alias` is the same either way.""" - payload_encoder: Callable[[Any], bytes] = _json_payload_encoder - payload_decoder: Callable[[bytes], Any] = _json_payload_decoder + payload_codec: PayloadCodec = MESSAGEPACK_PAYLOAD_CODEC schema_id: str | None = None schema_name: str | None = None schema_version: str | None = None @@ -245,8 +422,13 @@ class DocumentDefinition(Generic[T]): if self.validate: self.validate(value) tree = self.encode(value) - _refuse_past_depth_limit(tree, self.type, "write") - return self.payload_encoder(tree) + _check_payload_tree(tree, self.type, "write") + codec = self.payload_codec + try: + return codec.encode(tree) + except PayloadCodecFault as fault: + refusal = _fault_refusal(self.type, codec, "write", fault) + raise refusal def decode_payload(self, payload: bytes) -> T: return self.decode_payload_marked(payload)[0] @@ -254,7 +436,23 @@ class DocumentDefinition(Generic[T]): def decode_payload_marked(self, payload: bytes) -> tuple[T, UnreadSlots | None]: """The value and what the decode found past the declared slots, at any depth. The payload depth check runs first: a payload past the limit is refused, never marked.""" - tree = read_payload_tree(self.type, payload, self.payload_decoder) + codec = self.payload_codec + + def read(raw: bytes) -> Any: + try: + return codec.decode(raw) + except (msgpack.ExtraData, *codec.depth_faults): + raise + except PayloadCodecFault as fault: + refusal = _fault_refusal(self.type, codec, "read", fault) + except ValueError as fault: + refusal = CultCachePayloadFormatError(self.type, codec.name, "read", "not_one_value", type(fault).__name__) + raise refusal + + tree = read_payload_tree(self.type, payload, read, codec.depth_faults) + return self._decode_tree(tree) + + def _decode_tree(self, tree: Any) -> tuple[T, UnreadSlots | None]: outer, opened = _current_scope(), _SlotScope() _scope.open = opened try: @@ -325,8 +523,7 @@ def define_document_type( name: str | Extractor[T] | None = None, indexes: dict[str, str | Extractor[T]] | None = None, roles: dict[str, str | Extractor[T]] | None = None, - payload_encoder: Callable[[Any], bytes] = _json_payload_encoder, - payload_decoder: Callable[[bytes], Any] = _json_payload_decoder, + payload_codec: PayloadCodec | None = None, schema_id: str | None = None, schema_name: str | None = None, schema_version: str | None = None, @@ -335,6 +532,10 @@ def define_document_type( compatible_schema_ids: Iterable[str] | None = None, members: Iterable[CultCacheSchemaCatalogMember] | Callable[[], tuple[CultCacheSchemaCatalogMember, ...]] | None = None, ) -> DocumentDefinition[T]: + """Define a document type. Its payload is the MessagePack of what `encode` returns; pass + `payload_codec=JSON_PAYLOAD_CODEC` to read a store written by cultcache-py 0.3 or earlier, and `convert` to write + it as a new MessagePack store.""" + codec = {} if payload_codec is None else {"payload_codec": payload_codec} return DocumentDefinition( type=type, encode=encode, @@ -344,8 +545,7 @@ def define_document_type( name=name, indexes=indexes or {}, roles=roles or {}, - payload_encoder=payload_encoder, - payload_decoder=payload_decoder, + **codec, schema_id=schema_id, schema_name=schema_name, schema_version=schema_version, @@ -410,8 +610,6 @@ def define_database_entry_type( name=name, indexes=indexes, roles=roles, - payload_encoder=_pack_tree, - payload_decoder=_messagepack_payload_decoder, schema_id=schema_id, schema_name=schema_name, schema_version=schema_version, diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index 99e38d0d..387a5145 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -17,7 +17,7 @@ from typing import Any, Callable, Iterator, TypeVar import msgpack -from .documents import UnreadSlots +from .documents import UnreadSlots, _unpack_tree from .backing_store import ( CultCacheEnvelope, CultCacheSchemaCatalogEntry, @@ -144,8 +144,8 @@ class JsonLinesBackingStore: change(existing) self._replace_all(list(existing.values())) - def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) + @staticmethod + def _file_bytes(envelopes: list[CultCacheEnvelope]) -> bytes: lines = [] for envelope in sorted(envelopes, key=lambda item: (item.type, item.key)): lines.append(json.dumps({ @@ -154,7 +154,11 @@ class JsonLinesBackingStore: "payload": base64.b64encode(envelope.payload).decode("ascii"), "stored_at": envelope.stored_at, }, ensure_ascii=False, sort_keys=True)) - commit_store_file(self.path, ("\n".join(lines) + ("\n" if lines else "")).encode("utf-8")) + return ("\n".join(lines) + ("\n" if lines else "")).encode("utf-8") + + def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + commit_store_file(self.path, self._file_bytes(envelopes)) class SingleFileMessagePackBackingStore: @@ -181,7 +185,7 @@ class SingleFileMessagePackBackingStore: if data is None: return None try: - decoded = msgpack.unpackb(data, raw=False) + decoded = _unpack_tree(data) snapshot = _decode_snapshot(decoded) if snapshot is not None: self._format, envelopes = snapshot @@ -233,6 +237,10 @@ class SingleFileMessagePackBackingStore: with store_write_lock(self.path, self.lock_timeout): yield + def _file_bytes(self, envelopes: list[CultCacheEnvelope]) -> bytes: + """The bytes of a new store holding exactly `envelopes`, under the header this store read (as a write keeps it).""" + return msgpack.packb(_encode_snapshot(envelopes, [], self._format)[0], use_bin_type=True) # type: ignore[no-any-return] + def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> dict[str, str]: snapshot, stamps = _encode_snapshot(supplied, arrived, self._format) commit_store_file(self.path, msgpack.packb(snapshot, use_bin_type=True)) @@ -669,6 +677,9 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]: + """A legacy envelope holds its payload as an inline MessagePack subtree or as bytes. The store's reader judged the + subtree's map keys when it unpacked the whole file (`_unpack_tree`, whatever the record's type or codec), so a nil or + bin key anywhere in the file refuses the store; the registered codec then reads the repacked bytes or refuses them.""" if not isinstance(decoded, list): raise ValueError("CultCache MessagePack store is not a recognized snapshot") envelopes: list[CultCacheEnvelope] = [] diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 7ff4e297..a950be12 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -1,9 +1,12 @@ from __future__ import annotations +import base64 +import json import os import tempfile import threading import unittest +import msgpack from dataclasses import asdict, dataclass, replace from pathlib import Path from cultcache_py.backing_store import CultCacheEnvelope @@ -120,11 +123,6 @@ class CultCacheTests(unittest.TestCase): self.assertIsNone(cache.get_key_by_index(document, "category", "Rare")) def test_database_entry_formatter_uses_slot_indexed_messagepack_array(self) -> None: - try: - import msgpack # type: ignore - except ModuleNotFoundError: - self.skipTest("msgpack optional dependency is not installed") - @dataclass class Settings: theme: str @@ -159,11 +157,6 @@ class CultCacheTests(unittest.TestCase): self.assertEqual([member.slot for member in first.members], [0, 2]) def test_messagepack_store_writes_v1_snapshot(self) -> None: - try: - import msgpack # type: ignore - except ModuleNotFoundError: - self.skipTest("msgpack optional dependency is not installed") - with tempfile.TemporaryDirectory() as tmp: document = define_database_entry_type( "settings", @@ -1570,6 +1563,39 @@ class CultCacheTests(unittest.TestCase): SingleFileMessagePackBackingStore(path).pull_all() self.assertNotIsInstance(caught.exception, FileNotFoundError) + def test_json_lines_pull_all_reads_hand_written_lines_and_names_a_malformed_one(self) -> None: + def line(**fields: object) -> str: + return json.dumps(fields, ensure_ascii=False) + + first = base64.b64encode(b"one").decode("ascii") + second = base64.b64encode(b"two").decode("ascii") + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "hand.jsonl" + lines = [ + line(key="é-ключ", type="tests.t", payload=first, storedAt="2026-01-01T00:00:00Z", schemaId="schema.one"), + "", + line(key="k2", type="tests.t", payload=second, stored_at="2026-02-02T00:00:00Z", schema_id="schema.two"), + ] + path.write_bytes(("\n".join(lines) + "\n").encode("utf-8")) + pulled = JsonLinesBackingStore(path).pull_all() + self.assertEqual( + [(e.key, e.type, e.payload, e.stored_at, e.schema_id) for e in pulled], + [ + ("é-ключ", "tests.t", b"one", "2026-01-01T00:00:00Z", "schema.one"), + ("k2", "tests.t", b"two", "2026-02-02T00:00:00Z", "schema.two"), + ], + ) + written = Path(tmp) / "written.jsonl" + JsonLinesBackingStore(written).push(CultCacheEnvelope.create(key="k", type="tests.t", payload=b"x")) + raw = written.read_bytes() + self.assertTrue(raw.endswith(b"\n")) + self.assertNotIn(b"\r", raw) + broken = Path(tmp) / "broken.jsonl" + broken.write_bytes(("\n".join([lines[0], "", line(key="k3", type="tests.t")]) + "\n").encode("utf-8")) + with self.assertRaises(ValueError) as caught: + JsonLinesBackingStore(broken).pull_all() + self.assertEqual(str(caught.exception), f"Malformed CultCache JSONL envelope at {broken}:3: missing 'payload'") + # The element-id marker: tests/vectors/document-variants-c2a/v3-base.msgpack is v1-base with its header replaced. _C2A_VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "document-variants-c2a" diff --git a/packages/cultcache-py/tests/test_payload_codec.py b/packages/cultcache-py/tests/test_payload_codec.py new file mode 100644 index 00000000..bd1720d9 --- /dev/null +++ b/packages/cultcache-py/tests/test_payload_codec.py @@ -0,0 +1,1065 @@ +from __future__ import annotations + +import collections +import contextlib +import inspect +import json +import os +import shutil +import tempfile +import traceback +import unittest +from pathlib import Path +from unittest import mock + +import msgpack + +from cultcache_py import ( + JSON_PAYLOAD_CODEC, + MESSAGEPACK_PAYLOAD_CODEC, + CultCache, + CultCacheEnvelope, + CultCachePayloadDepthError, + CultCachePayloadFormatError, + DocumentDefinition, + JsonLinesBackingStore, + PayloadCodec, + PayloadCodecFault, + SingleFileMessagePackBackingStore, + define_database_entry_type, + define_document_type, + convert, +) +from cultcache_py.cache import CultCacheError +from cultcache_py.stores import StoreUnreadableError + +CANARY = "CANARY-payload-codec-5b81e0" +LEGACY_JSON = b'{"n": 1, "schema": "x"}' +KEY_KINDS = "str, int, float, bool or timestamp keys" + +# The rows of the C# reference's object slot (map P2, jobK): the bytes of one map. +ACCEPTED_KEY_ROWS = { + "int": "810101", + "negint": "81ff01", + "u64max": "81cfffffffffffffffff01", + "i64min": "81d3800000000000000001", + "f32": "81ca3fc0000001", + "f64": "81cb3ff800000000000001", + "bool": "81c301", + "tskey": "81d6ff0000000001", + "str-and-int": "82a1610102 02".replace(" ", ""), +} +REFUSED_KIND_ROWS = { + "nil": ("81c001", "nil"), + "bin": ("81c4016b01", "bin"), + "array": ("81910101", "array"), + "map": ("8181010101", "map"), + "ext": ("81d4010001", "ext"), +} +DUPLICATE_ROWS = { + "dup-int": "82010101 02".replace(" ", ""), + "dup-str": "82a16101a16102", + "fix1-u8-1": "8201 01cc0102".replace(" ", ""), + "int1-f64-1": "820101cb3ff000000000000002", + "int1-true": "820101c302", +} + + +def _store_with(path: Path, payload: bytes) -> SingleFileMessagePackBackingStore: + store = SingleFileMessagePackBackingStore(path) + store.push(CultCacheEnvelope.create(key="k", type="tests.legacy", payload=payload)) + return store + + +def _chain_text(error: BaseException) -> str: + return "".join(traceback.format_exception(error)) + str(error) + repr(error) + + +class PayloadCodecTests(unittest.TestCase): + def setUp(self) -> None: + self.directory = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.directory, True) + + def _cache(self, document, store) -> CultCache: + return CultCache.builder().register_document_type(document).add_generic_store(store).build() + + def _refused(self, call, *args) -> CultCachePayloadFormatError: + with self.assertRaises(CultCachePayloadFormatError) as caught: + call(*args) + return caught.exception + + def test_the_default_is_messagepack_for_both_definitions(self) -> None: + document = define_document_type("tests.x") + self.assertIs(document.payload_codec, MESSAGEPACK_PAYLOAD_CODEC) + self.assertEqual(document.encode_payload({"a": [1]}), bytes.fromhex("81a1619101")) + self.assertIs(define_database_entry_type("tests.entry", [("a", 0)]).payload_codec, MESSAGEPACK_PAYLOAD_CODEC) + + def test_the_default_has_one_author(self) -> None: + self.assertIs(DocumentDefinition("tests.x").payload_codec, MESSAGEPACK_PAYLOAD_CODEC) + self.assertIs(define_document_type("tests.x").payload_codec, MESSAGEPACK_PAYLOAD_CODEC) + self.assertIs(define_document_type("tests.x", payload_codec=JSON_PAYLOAD_CODEC).payload_codec, JSON_PAYLOAD_CODEC) + self.assertIsNone(inspect.signature(define_document_type).parameters["payload_codec"].default) + + def test_the_json_codec_writes_the_0_3_bytes(self) -> None: + written = JSON_PAYLOAD_CODEC.encode({"b": "é", "a": [1, 2]}) + self.assertEqual(written, b'{"a": [1, 2], "b": "\xc3\xa9"}') + self.assertEqual(JSON_PAYLOAD_CODEC.decode(written), {"a": [1, 2], "b": "é"}) + self.assertEqual(JSON_PAYLOAD_CODEC.encode({1: "a", 3: 2**70}), b'{"1": "a", "3": 1180591620717411303424}') + + def test_a_json_store_is_refused_typed_and_loads_nothing(self) -> None: + store = _store_with(self.directory / "legacy.cc", LEGACY_JSON) + document = define_document_type("tests.legacy") + cache = self._cache(document, store) + with self.assertRaises(ValueError) as caught: + cache.pull_all_backing_stores() + message = str(caught.exception) + self.assertEqual(type(caught.exception), ValueError) + self.assertIn("tests.legacy payload has", message) + self.assertIn("bytes after its first MessagePack value (refused on read)", message) + self.assertNotIn("schema", message) + self.assertEqual(cache.get_all(document), []) + + def test_the_refusal_texts_are_exact_with_their_codes(self) -> None: + count = len(LEGACY_JSON) - 1 + with self.assertRaises(ValueError) as read: + define_document_type("tests.text").decode_payload(LEGACY_JSON) + self.assertEqual(type(read.exception), ValueError) + self.assertEqual(str(read.exception), f"tests.text payload has {count} bytes after its first MessagePack value (refused on read).") + with self.assertRaises(ValueError) as entry: + define_database_entry_type("tests.entry", [("a", 0)]).decode_payload(LEGACY_JSON) + self.assertEqual(str(entry.exception), f"tests.entry payload has {count} bytes after its first MessagePack value (refused on read).") + json_side = self._refused(define_document_type("tests.text", payload_codec=JSON_PAYLOAD_CODEC).decode_payload, b"\x81\xa1a") + self.assertEqual(str(json_side), "tests.text payload is not one JSON value (UnicodeDecodeError); refused on read.") + self.assertEqual((json_side.type_name, json_side.codec, json_side.direction), ("tests.text", "JSON", "read")) + nil = self._refused(define_document_type("tests.text").decode_payload, bytes.fromhex("81c001")) + self.assertEqual( + str(nil), + "tests.text payload has a map key of kind nil; MessagePack payloads take str, int, float, bool or timestamp keys (refused on read).", + ) + self.assertEqual((nil.code, nil.direction), ("map_key_kind", "read")) + duplicate = self._refused(define_document_type("tests.text").decode_payload, bytes.fromhex("82010101 02".replace(" ", ""))) + self.assertEqual(str(duplicate), "tests.text payload has two map keys Python holds as one; refused on read.") + self.assertEqual(duplicate.code, "duplicate_map_key") + key = self._refused(define_document_type("tests.text", encode=lambda value: value).encode_payload, {b"k": 1}) + self.assertEqual( + str(key), + "tests.text payload has a map key of kind bin; MessagePack payloads take str, int, float, bool or timestamp keys (refused on write).", + ) + self.assertEqual((key.type_name, key.codec, key.direction, key.code), ("tests.text", "MessagePack", "write", "map_key_kind")) + big = self._refused(define_document_type("tests.text", encode=lambda value: value).encode_payload, {"a": 2**64}) + self.assertEqual(str(big), "tests.text payload has an integer beyond 64 bits; refused on write.") + self.assertEqual((big.code, big.direction), ("integer_out_of_range", "write")) + + def test_a_consumer_codec_fault_with_its_own_code_is_a_typed_refusal_naming_it(self) -> None: + def encode(tree: object) -> bytes: + raise PayloadCodecFault("no_sets", "set") + + document = define_document_type("tests.own", payload_codec=PayloadCodec("Own", encode, lambda payload: None)) + refusal = self._refused(document.encode_payload, {"a": 1}) + self.assertEqual(str(refusal), "tests.own payload is refused by the Own codec (no_sets); refused on write.") + self.assertEqual((refusal.codec, refusal.code, refusal.direction), ("Own", "no_sets", "write")) + fault = PayloadCodecFault("no_sets", "set") + self.assertEqual((fault.code, fault.kind), ("no_sets", "set")) + self.assertIsInstance(fault, ValueError) + + def test_a_json_store_opted_in_reads_and_convert_writes_it_anew(self) -> None: + path = self.directory / "legacy.cc" + store = _store_with(path, LEGACY_JSON) + legacy = define_document_type("tests.legacy", payload_codec=JSON_PAYLOAD_CODEC) + old = self._cache(legacy, store) + old.pull_all_backing_stores() + self.assertEqual(old.get(legacy, "k"), {"n": 1, "schema": "x"}) + modern = define_document_type("tests.legacy") + destination = self.directory / "converted.cc" + original = path.read_bytes() + self.assertEqual(convert(store, destination, [modern]), 1) + self.assertEqual(path.read_bytes(), original) + reread = self._cache(modern, SingleFileMessagePackBackingStore(destination)) + reread.pull_all_backing_stores() + self.assertEqual(reread.get(modern, "k"), {"n": 1, "schema": "x"}) + self.assertEqual(reread.get_envelope(modern, "k").payload, MESSAGEPACK_PAYLOAD_CODEC.encode({"n": 1, "schema": "x"})) + + def test_wrong_format_payloads_are_typed_refusals_on_both_sides(self) -> None: + document = define_document_type("tests.faults") + for payload in (b"[1,2]", b'"hi"', b"true", b"null", b"1.5", b"-3", b"12", b"\x93\x01\x02", b"\xc1", b"\xa2\xff\xfe"): + with self.subTest(payload=payload): + with self.assertRaises(ValueError): + document.decode_payload(payload) + json_document = define_document_type("tests.faults", payload_codec=JSON_PAYLOAD_CODEC) + caught = self._refused(json_document.decode_payload, MESSAGEPACK_PAYLOAD_CODEC.encode({"a": 1})) + self.assertNotIn("JSON_PAYLOAD_CODEC", str(caught)) + self.assertIn("JSON", str(caught)) + + def test_a_one_digit_json_text_reads_as_a_messagepack_integer(self) -> None: + self.assertEqual(define_document_type("tests.digit").decode_payload(b"5"), 53) + + def test_depth_faults_belong_to_the_codec(self) -> None: + def deep(payload: bytes) -> object: + raise RecursionError + + declaring = define_document_type("tests.d", payload_codec=PayloadCodec("t", lambda tree: b"", deep, depth_faults=(RecursionError,))) + with self.assertRaises(CultCachePayloadDepthError): + declaring.decode_payload(b"x") + silent = define_document_type("tests.d", payload_codec=PayloadCodec("t", lambda tree: b"", deep)) + with self.assertRaises(RecursionError) as caught: + silent.decode_payload(b"x") + self.assertNotIsInstance(caught.exception, CultCachePayloadDepthError) + + def test_a_codec_decode_fault_that_is_a_value_error_is_a_format_error_naming_its_class(self) -> None: + def refuse(payload: bytes) -> object: + raise KeyError(CANARY) + + document = define_document_type("tests.v", payload_codec=PayloadCodec("t", lambda tree: b"", lambda payload: int("x"))) + self.assertIn("(ValueError)", str(self._refused(document.decode_payload, b"x"))) + passthrough = define_document_type("tests.v", payload_codec=PayloadCodec("t", lambda tree: b"", refuse)) + with self.assertRaises(KeyError): + passthrough.decode_payload(b"x") + + def test_a_refused_key_reaches_no_store(self) -> None: + document = define_document_type("tests.keys", encode=lambda value: value) + store = SingleFileMessagePackBackingStore(self.directory / "keys.cc") + cache = self._cache(document, store) + with self.assertRaises(CultCachePayloadFormatError): + cache.put(document, "k", {b"k": "a"}) + self.assertEqual(store.pull_all(), []) + + def test_the_library_codecs_refuse_nodes_the_depth_walk_cannot_see(self) -> None: + class OnlyIter: + def __iter__(self): + return iter([1]) + + nodes = (OnlyIter(), collections.deque([1]), collections.UserDict(a=1), frozenset([1])) + for codec in (MESSAGEPACK_PAYLOAD_CODEC, JSON_PAYLOAD_CODEC): + document = define_document_type("tests.unseen", encode=lambda value: value, payload_codec=codec) + for node in nodes: + with self.subTest(codec=codec.name, node=type(node).__name__): + with self.assertRaises(TypeError): + document.encode_payload({"a": node}) + + # The key rule is the C# reference's (map P2), both ways. + + def test_reference_key_rule_reads_through_an_any_slot(self) -> None: + entry = define_database_entry_type("tests.any", [("v", 0)]) + for name, row in ACCEPTED_KEY_ROWS.items(): + with self.subTest(row=name): + value = entry.decode_payload(b"\x91" + bytes.fromhex(row))["v"] + again = entry.decode_payload(entry.encode_payload({"v": value}))["v"] + self.assertEqual(again, value) + self.assertEqual([type(key) for key in again], [type(key) for key in value]) + sixty_four = entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["u64max"]))["v"] + self.assertEqual(list(sixty_four), [2**64 - 1]) + self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["i64min"]))["v"]), [-(2**63)]) + self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["bool"]))["v"]), [True]) + self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["f64"]))["v"]), [1.5]) + self.assertEqual(list(entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["tskey"]))["v"]), [msgpack.Timestamp(0, 0)]) + mixed = entry.decode_payload(b"\x91" + bytes.fromhex(ACCEPTED_KEY_ROWS["str-and-int"]))["v"] + self.assertEqual(mixed, {"a": 1, 2: 2}) + for name, (row, kind) in REFUSED_KIND_ROWS.items(): + with self.subTest(row=name): + refusal = self._refused(entry.decode_payload, b"\x91" + bytes.fromhex(row)) + self.assertEqual((refusal.code, refusal.direction), ("map_key_kind", "read")) + self.assertIn(f"map key of kind {kind};", str(refusal)) + for name, row in DUPLICATE_ROWS.items(): + with self.subTest(row=name): + refusal = self._refused(entry.decode_payload, b"\x91" + bytes.fromhex(row)) + self.assertEqual(refusal.code, "duplicate_map_key") + + def test_reference_key_rule_writes_and_reads_back_through_a_document_type(self) -> None: + document = define_document_type("tests.keys", encode=lambda value: value) + for tree in ({1: "a"}, {True: 1}, {1.5: 1}, {msgpack.Timestamp(0, 0): 1}, {"a": {2: 3}}, {"a": [{1.5: 1}]}, {-(2**63): 1}, {2**64 - 1: 1}): + with self.subTest(tree=tree): + self.assertEqual(document.decode_payload(document.encode_payload(tree)), tree) + refused = ( + ({b"k": 1}, "bin"), + ({None: 1}, "nil"), + ({(1,): 1}, "array"), + ({frozenset(): 1}, "array"), + ({msgpack.ExtType(1, b"x"): 1}, "ext"), + ({object(): 1}, "unsupported"), + ({"a": [{b"k": 1}]}, "bin"), + ) + for tree, kind in refused: + with self.subTest(tree=tree): + refusal = self._refused(document.encode_payload, tree) + self.assertEqual((refusal.code, refusal.direction), ("map_key_kind", "write")) + self.assertIn(f"map key of kind {kind};", str(refusal)) + legacy = define_document_type("tests.keys", encode=lambda value: value, payload_codec=JSON_PAYLOAD_CODEC) + self.assertEqual(legacy.encode_payload({1: "a"}), b'{"1": "a"}') + + def test_a_float32_key_writes_as_a_float_the_reference_reads(self) -> None: + from cultcache_py.leaves import Float32 + + written = MESSAGEPACK_PAYLOAD_CODEC.encode({Float32(1.5): 1}) + self.assertEqual(written, bytes.fromhex("81ca3fc0000001")) + self.assertEqual(MESSAGEPACK_PAYLOAD_CODEC.decode(written), {1.5: 1}) + + def test_the_codec_owns_its_keys_both_ways(self) -> None: + codec = MESSAGEPACK_PAYLOAD_CODEC + for tree, code, kind in ( + ({b"k": 1}, "map_key_kind", "bin"), + ({None: 1}, "map_key_kind", "nil"), + ({"a": 2**64}, "integer_out_of_range", ""), + ): + with self.subTest(tree=tree): + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(tree) + self.assertEqual((caught.exception.code, caught.exception.kind), (code, kind)) + for name, (row, kind) in REFUSED_KIND_ROWS.items(): + with self.subTest(row=name): + with self.assertRaises(PayloadCodecFault) as caught: + codec.decode(bytes.fromhex(row)) + self.assertEqual((caught.exception.code, caught.exception.kind), ("map_key_kind", kind)) + for name, row in DUPLICATE_ROWS.items(): + with self.subTest(row=name): + with self.assertRaises(PayloadCodecFault) as caught: + codec.decode(bytes.fromhex(row)) + self.assertEqual((caught.exception.code, caught.exception.kind), ("duplicate_map_key", "")) + for tree in ({1: "a"}, {True: 1}, {1.5: [2**64 - 1, -(2**63)]}, {"a": {-1: None}}, [{2: 3}]): + with self.subTest(tree=tree): + self.assertEqual(codec.decode(codec.encode(tree)), tree) + + def test_the_codec_carries_no_extension_type_but_the_timestamp(self) -> None: + codec = MESSAGEPACK_PAYLOAD_CODEC + ext = msgpack.ExtType(1, b"x") + for name, tree in (("root", ext), ("map-value", {"a": ext}), ("nested-item", [[ext]])): + with self.subTest(encode=name): + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(tree) + self.assertEqual((caught.exception.code, caught.exception.kind), ("value_kind", "ext")) + for name, row in (("root", "d40100"), ("map-value", "81a161d40100"), ("nested-item", "91d40100")): + with self.subTest(decode=name): + with self.assertRaises(PayloadCodecFault) as caught: + codec.decode(bytes.fromhex(row)) + self.assertEqual((caught.exception.code, caught.exception.kind), ("value_kind", "ext")) + document = define_document_type("tests.ext", encode=lambda value: value) + text = "tests.ext payload holds a value of kind ext; MessagePack payloads carry no extension type but the timestamp (refused on {})." + self.assertEqual(str(self._refused(document.encode_payload, ext)), text.format("write")) + refusal = self._refused(document.decode_payload, bytes.fromhex("91d40100")) + self.assertEqual((refusal.code, str(refusal)), ("value_kind", text.format("read"))) + + def test_mixed_width_nan_keys_are_a_documented_refusal(self) -> None: + # The bytes C# accepts for a map holding float.NaN and double.NaN keys (two keys); it writes the values as int32. + row = "82 ca ffc00000 01 cb fff8000000000000 02" + written = "82 ca ffc00000 d2 00000001 cb fff8000000000000 d2 00000002" + with self.assertRaises(PayloadCodecFault) as caught: + MESSAGEPACK_PAYLOAD_CODEC.decode(bytes.fromhex(row)) + self.assertEqual((caught.exception.code, caught.exception.kind), ("duplicate_map_key", "")) + document = define_document_type("tests.nan", encode=lambda value: value) + self.assertEqual(self._refused(document.decode_payload, bytes.fromhex(row)).code, "duplicate_map_key") + readme = (Path(__file__).resolve().parents[1] / "README.md").read_text(encoding="utf-8") + section = readme.split("## Wire Parity", 1)[1].split("\n## ", 1)[0] + self.assertIn("duplicate_map_key", section) + self.assertIn(row, section) + self.assertIn(written, section) + with self.assertRaises(PayloadCodecFault) as caught: + MESSAGEPACK_PAYLOAD_CODEC.decode(bytes.fromhex(written)) + self.assertEqual(caught.exception.code, "duplicate_map_key") + + def test_every_nan_key_is_one_key_both_ways(self) -> None: + from cultcache_py.leaves import Float32 + + codec = MESSAGEPACK_PAYLOAD_CODEC + collisions = ( + {float("nan"): 1, float("nan"): 2}, + {Float32(float("nan")): 1, Float32(float("nan")): 2}, + {float("nan"): 1, Float32(float("nan")): 2}, + ) + for tree in collisions: + for wrapped in (tree, {"n": [tree]}): + with self.subTest(tree=wrapped): + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(wrapped) + self.assertEqual((caught.exception.code, caught.exception.kind), ("duplicate_map_key", "")) + for name, row in ( + ("f64-pair", "82cb7ff800000000000001cb7ff800000000000002"), + ("f32-pair", "82ca7fc0000001ca7fc0000002"), + ("mixed-pair", "82cb7ff800000000000001ca7fc0000002"), + ): + with self.subTest(row=name): + with self.assertRaises(PayloadCodecFault) as caught: + codec.decode(bytes.fromhex(row)) + self.assertEqual((caught.exception.code, caught.exception.kind), ("duplicate_map_key", "")) + for single in (float("nan"), Float32(float("nan"))): + with self.subTest(single=type(single).__name__): + (key,) = codec.decode(codec.encode({single: 1})) + self.assertNotEqual(key, key) + + def test_a_float32_key_beside_a_key_the_reader_holds_equal_is_refused_on_write(self) -> None: + from cultcache_py.leaves import Float32 + + codec = MESSAGEPACK_PAYLOAD_CODEC + collisions = ( + {1: "a", Float32(1.0): "b"}, + {True: "a", Float32(1.0): "b"}, + {1.5: "a", Float32(1.5): "b"}, + {0: "a", Float32(-0.0): "b"}, + ) + for tree in collisions: + for wrapped in (tree, {"n": [tree]}): + with self.subTest(tree=wrapped): + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(wrapped) + self.assertEqual((caught.exception.code, caught.exception.kind), ("duplicate_map_key", "")) + document = define_document_type("tests.f32", encode=lambda value: value) + store = SingleFileMessagePackBackingStore(self.directory / "f32.cc") + cache = self._cache(document, store) + with self.assertRaises(CultCachePayloadFormatError) as caught: + cache.put(document, "k", collisions[0]) + refusal = caught.exception + self.assertEqual((refusal.code, refusal.direction), ("duplicate_map_key", "write")) + self.assertIn("refused on write", str(refusal)) + self.assertFalse(store.path.exists()) + for tree in ( + {0.1: "a", Float32(0.1): "b"}, + {1: "a", Float32(1.5): "b"}, + {"a": 1, Float32(1.0): 2}, + ): + with self.subTest(accepted=tree): + self.assertEqual(len(codec.decode(codec.encode(tree))), 2) + + def test_the_codec_write_bound_is_the_payload_bound(self) -> None: + from cultcache_py.leaves import Float32 + + def lists(depth: int, leaf: object = 1) -> object: + tree = leaf + for _ in range(depth): + tree = [tree] + return tree + + def maps(depth: int, leaf: object = 1) -> object: + tree = leaf + for _ in range(depth): + tree = {"a": tree} + return tree + + codec = MESSAGEPACK_PAYLOAD_CODEC + for build in (lists, maps): + for leaf in (1, Float32(1.5)): + with self.subTest(build=build.__name__, leaf=leaf): + self.assertIsNotNone(codec.decode(codec.encode(build(256, leaf)))) + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(build(257, leaf)) + self.assertEqual((caught.exception.code, caught.exception.kind), ("depth_limit", "")) + document = define_document_type("tests.depth", encode=lambda value: value) + with self.assertRaises(CultCachePayloadDepthError): + document.encode_payload(lists(257)) + with self.assertRaises(CultCachePayloadDepthError): + document.decode_payload(b"\x91" * 257 + b"\x01") + self.assertEqual(document.decode_payload(b"\x91" * 256 + b"\x01"), lists(256)) + + def test_the_codec_read_bound_is_the_payload_bound(self) -> None: + codec = MESSAGEPACK_PAYLOAD_CODEC + document = define_document_type("tests.depth", encode=lambda value: value) + for name, unit, leaf in (("lists", b"\x91", b"\x01"), ("maps", b"\x81\xa1a", b"\x01")): + with self.subTest(shape=name): + self.assertIsNotNone(codec.decode(unit * 256 + leaf)) + with self.assertRaises(PayloadCodecFault) as caught: + codec.decode(unit * 257 + leaf) + self.assertEqual((caught.exception.code, caught.exception.kind), ("depth_limit", "")) + with self.assertRaises(CultCachePayloadDepthError): + document.decode_payload(unit * 257 + leaf) + + def test_timestamps_lie_within_datetime_both_ways(self) -> None: + codec = MESSAGEPACK_PAYLOAD_CODEC + stamp = msgpack.Timestamp + for ts in (stamp(-62135596800, 0), stamp(253402300799, 999999999)): + for tree in (ts, {"a": ts}, {ts: 1}): + with self.subTest(inside=tree): + self.assertEqual(codec.decode(codec.encode(tree)), tree) + document = define_document_type("tests.ts", encode=lambda value: value) + text = "tests.ts payload holds a timestamp outside years 1 to 9999; refused on {}." + for ts in (stamp(-62135596801, 999999999), stamp(253402300800, 0)): + for tree in (ts, {"a": ts}, {ts: 1}): + with self.subTest(outside=tree): + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(tree) + self.assertEqual((caught.exception.code, caught.exception.kind), ("timestamp_out_of_range", "")) + packed = msgpack.packb(tree, use_bin_type=True) + with self.assertRaises(PayloadCodecFault) as caught: + codec.decode(packed) + self.assertEqual((caught.exception.code, caught.exception.kind), ("timestamp_out_of_range", "")) + self.assertEqual(str(self._refused(document.encode_payload, tree)), text.format("write")) + self.assertEqual(str(self._refused(document.decode_payload, packed)), text.format("read")) + + def test_the_integer_range_is_closed_at_both_ends_at_the_root_too(self) -> None: + codec = MESSAGEPACK_PAYLOAD_CODEC + for value in (2**64 - 1, -(2**63), 0): + with self.subTest(accepted=value): + self.assertEqual(codec.decode(codec.encode(value)), value) + for value in (2**64, -(2**63) - 1): + with self.subTest(refused=value): + with self.assertRaises(PayloadCodecFault) as caught: + codec.encode(value) + self.assertEqual(caught.exception.code, "integer_out_of_range") + + def test_a_consumer_codec_that_refuses_a_nesting_gets_the_depth_refusal_naming_the_direction(self) -> None: + def encode(tree: object) -> bytes: + raise PayloadCodecFault("depth_limit") + + def decode(payload: bytes) -> object: + raise PayloadCodecFault("depth_limit") + + document = define_document_type("tests.own_depth", encode=lambda value: value, payload_codec=PayloadCodec("Own", encode, decode)) + for direction, call, argument in (("write", document.encode_payload, {"a": 1}), ("read", document.decode_payload, b"x")): + with self.subTest(direction=direction): + with self.assertRaises(CultCachePayloadDepthError) as caught: + call(argument) + self.assertEqual( + str(caught.exception), + f"tests.own_depth payload nests 257 containers deep; the limit is 256 (refused on {direction}).", + ) + + def test_a_codec_fault_prints_its_code_only(self) -> None: + self.assertEqual(str(PayloadCodecFault("some_code", "some_kind")), "some_code") + self.assertEqual(str(PayloadCodecFault("some_code")), "some_code") + self.assertEqual(PayloadCodecFault("some_code").kind, "") + + def test_define_document_type_forwards_every_schema_field(self) -> None: + document = define_document_type( + "tests.schema", + schema_id="sid", + schema_name="sname", + schema_version="sname.v9", + content_hash="hash", + canonical_schema_json='{"canonical":true}', + compatible_schema_ids=["sid", "older"], + ) + self.assertEqual( + (document.schema_id, document.schema_name, document.schema_version, document.content_hash), + ("sid", "sname", "sname.v9", "hash"), + ) + self.assertEqual((document.canonical_schema_json, document.compatible_schema_ids), ('{"canonical":true}', ("sid", "older"))) + bare = define_document_type("tests.schema") + self.assertEqual((bare.schema_id, bare.compatible_schema_ids), (None, ())) + + def test_a_cyclic_tree_is_not_walked_forever_by_the_codec(self) -> None: + cycle: dict = {} + cycle["a"] = cycle + with self.assertRaises(PayloadCodecFault) as caught: + MESSAGEPACK_PAYLOAD_CODEC.encode(cycle) + self.assertEqual(caught.exception.code, "depth_limit") + with self.assertRaises(CultCachePayloadDepthError): + define_document_type("tests.cycle", encode=lambda value: value).encode_payload(cycle) + + # Integers beyond 64 bits are a format limit with a typed refusal. + + def test_integers_beyond_64_bits_are_refused_typed_on_write(self) -> None: + document = define_document_type("tests.big", encode=lambda value: value) + store = SingleFileMessagePackBackingStore(self.directory / "big.cc") + cache = self._cache(document, store) + for value in (2**64, -(2**63) - 1, 2**64 + 987654): + for name, tree in (("value", {"a": value}), ("list", {"a": [[value]]}), ("key", {value: 1}), ("root", value)): + with self.subTest(value=value, place=name): + refusal = self._refused(document.encode_payload, tree) + self.assertEqual((refusal.code, refusal.direction), ("integer_out_of_range", "write")) + text = _chain_text(refusal) + self.assertNotIn(str(value), text) + self.assertNotIn(str(value)[:10], text) + with self.assertRaises(CultCachePayloadFormatError): + cache.put(document, "k", {"a": value}) + self.assertEqual(store.pull_all(), []) + for value in (2**64 - 1, -(2**63)): + for tree in ({"a": value}, {"a": [value]}, {value: 1}): + with self.subTest(value=value, tree=tree): + self.assertEqual(document.decode_payload(document.encode_payload(tree)), tree) + + # A refusal carries no payload byte or value, on its text or on its exception chain. + + def test_refusals_carry_no_payload_on_their_text_or_exception_chain(self) -> None: + raw = CANARY.encode() + text_key = bytes([0xA0 + len(raw)]) + raw + read_rows = { + "extra-data": (define_document_type("tests.c"), b"\x01" + raw), + "utf8": (define_document_type("tests.c", payload_codec=JSON_PAYLOAD_CODEC), b"\xff" + raw), + "json-text": (define_document_type("tests.c", payload_codec=JSON_PAYLOAD_CODEC), b'{"' + raw + b'": '), + "map-key-kind": (define_document_type("tests.c"), b"\x81\xc4" + bytes([len(raw)]) + raw + b"\x01"), + "duplicate-key": (define_document_type("tests.c"), b"\x82" + text_key + b"\x01" + text_key + b"\x02"), + "depth": (define_document_type("tests.c"), b"\x91" * 300 + text_key), + "not-msgpack": (define_document_type("tests.c"), b"\xc1" + raw), + } + for name, (document, payload) in read_rows.items(): + with self.subTest(row=name): + with self.assertRaises(ValueError) as caught: + document.decode_payload(payload) + self.assertIsNone(caught.exception.__context__) + self.assertIsNone(caught.exception.__cause__) + self.assertNotIn(CANARY, _chain_text(caught.exception)) + write = define_document_type("tests.c", encode=lambda value: value) + for name, tree in ( + ("map-key-kind", {raw: 1}), + ("integer", {"k": 2**64 + 987654, CANARY: 1}), + ("nested", {"a": [{raw: CANARY}]}), + ): + with self.subTest(row=name): + with self.assertRaises(CultCachePayloadFormatError) as caught: + write.encode_payload(tree) + self.assertIsNone(caught.exception.__context__) + self.assertIsNone(caught.exception.__cause__) + self.assertNotIn(CANARY, _chain_text(caught.exception)) + with self.assertRaises(CultCachePayloadDepthError) as deep: + define_document_type("tests.c", encode=lambda value: value).encode_payload(self._nested(300, CANARY)) + self.assertIsNone(deep.exception.__context__) + self.assertNotIn(CANARY, _chain_text(deep.exception)) + + @staticmethod + def _nested(depth: int, leaf: object) -> object: + tree = leaf + for _ in range(depth): + tree = [tree] + return tree + + # The legacy envelope reader extracts bytes; the registered codec still decides. + + def test_a_legacy_inline_payload_is_messagepack_and_a_json_definition_refuses_it(self) -> None: + path = self.directory / "inline.cc" + path.write_bytes( + msgpack.packb([{"key": "k", "type": "tests.legacy", "payload": {"n": 1}, "storedAt": "2026-01-01T00:00:00Z"}], use_bin_type=True) + ) + document = define_document_type("tests.legacy") + cache = self._cache(document, SingleFileMessagePackBackingStore(path)) + cache.pull_all_backing_stores() + self.assertEqual(cache.get(document, "k"), {"n": 1}) + json_document = define_document_type("tests.legacy", payload_codec=JSON_PAYLOAD_CODEC) + refused = self._cache(json_document, SingleFileMessagePackBackingStore(path)) + with self.assertRaises(CultCachePayloadFormatError) as caught: + refused.pull_all_backing_stores() + self.assertEqual(caught.exception.code, "not_one_value") + + def test_a_legacy_inline_payload_key_is_judged_by_the_store_reader_for_every_record(self) -> None: + def store_of(payload: object, name: str) -> SingleFileMessagePackBackingStore: + path = self.directory / f"inline-{name}.cc" + path.write_bytes( + msgpack.packb( + [{"key": "k", "type": "tests.legacy", "payload": payload, "storedAt": "2026-01-01T00:00:00Z"}], + use_bin_type=True, + ) + ) + return SingleFileMessagePackBackingStore(path) + + document = define_document_type("tests.legacy") + cache = self._cache(document, store_of({1: "a"}, "int")) + cache.pull_all_backing_stores() + self.assertEqual(cache.get(document, "k"), {1: "a"}) + with self.assertRaises(StoreUnreadableError) as caught: + store_of({None: "a"}, "nil").pull_all() + cause = caught.exception.__cause__ + self.assertIsInstance(cause, PayloadCodecFault) + self.assertEqual((cause.code, cause.kind), ("map_key_kind", "nil")) + + +MIGRATION_RECORDS = { + "k1": b'{"n": 1}', + "k2": b'["a", 2]', + "k3": b'{"s": "\xc3\xa9"}', + "k4": b'{"a": [1, 2], "b": {"c": true}}', + "k5": b'{"t": null, "f": 1.5}', + "k6": b'{"x": {"y": [[0, 1], []]}}', + "k7": b'{"big": 1180591620717411303424}', +} +BIG_DIGITS = "1180591620717411303424" +SOURCE_MISSING = "convert: argument source does not exist (source_missing); there is nothing to convert." +DESTINATION_EXISTS = ( + "convert: argument destination already exists (destination_exists); convert writes only a new store, " + "and the source is unchanged." +) + + +class ConvertTests(unittest.TestCase): + def setUp(self) -> None: + self.directory = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.directory, True) + + def _stores(self): + yield "single-file", lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc") + yield "jsonl", lambda name: JsonLinesBackingStore(self.directory / f"{name}.jsonl") + + def _filled(self, make, name: str, records: dict[str, bytes]): + store = make(name) + for key, payload in records.items(): + store.push(CultCacheEnvelope.create(key=key, type="tests.mig", payload=payload)) + return store + + def _pull_with(self, store, codec): + document = define_document_type("tests.mig", payload_codec=codec, encode=lambda value: value) + cache = CultCache.builder().register_document_type(document).add_generic_store(store).build() + cache.pull_all_backing_stores() + return cache.get_all(document) + + def test_convert_keeps_the_header_the_source_was_read_under(self) -> None: + vector = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "document-variants-c2a" / "v3-base.msgpack" + source_path = self.directory / "v3.cc" + source_path.write_bytes(vector.read_bytes()) + destination = self.directory / "out.cc" + self.assertEqual(convert(SingleFileMessagePackBackingStore(source_path), destination, []), 0) + header = msgpack.unpackb(destination.read_bytes(), raw=False)[0] + self.assertEqual(header, "cultcache.store.v3") + again = SingleFileMessagePackBackingStore(destination) + self.assertEqual(sorted(e.key for e in again.pull_all()), ["alpha", "beta"]) + self.assertEqual(again._format, "cultcache.store.v3") + + def test_convert_writes_a_new_store_of_the_same_kind(self) -> None: + records = {"k1": b"5", "k2": b"53", "k3": b'{"n": 1}'} + definition = define_document_type("tests.mig", encode=lambda value: value) + for label, make in self._stores(): + with self.subTest(store=label): + store = self._filled(make, f"src-{label}", records) + other = CultCacheEnvelope.create(key="o", type="tests.other", payload=b"opaque\x00bytes") + store.push(other) + destination = self.directory / f"dst-{label}.out" + self.assertEqual(convert(store, destination, [definition]), 3) + opened = type(store)(destination) + kept = next(item for item in opened.pull_all() if item.type == "tests.other") + self.assertEqual((kept.payload, kept.stored_at), (other.payload, other.stored_at)) + self.assertEqual( + {item.key: definition.decode_payload(item.payload) for item in opened.pull_all() if item.type == "tests.mig"}, + {"k1": 5, "k2": 53, "k3": {"n": 1}}, + ) + plain = self._filled(make, f"plain-{label}", records) + plain_destination = self.directory / f"plain-{label}.out" + convert(plain, plain_destination, [definition]) + self.assertEqual( + sorted(self._pull_with(type(store)(plain_destination), MESSAGEPACK_PAYLOAD_CODEC), key=str), [5, 53, {"n": 1}] + ) + raw = destination.read_bytes() + if label == "jsonl": + self.assertEqual(raw[:1], b"{") + else: + self.assertIsInstance(msgpack.unpackb(raw, raw=False, strict_map_key=False), list) + + def test_a_source_with_no_listed_record_still_writes_the_destination_with_count_zero(self) -> None: + for label, make in self._stores(): + with self.subTest(store=label): + store = self._filled(make, f"zero-{label}", {"k1": b'{"n": 1}'}) + destination = self.directory / f"zero-{label}.out" + self.assertEqual(convert(store, destination, [define_document_type("tests.nothing")]), 0) + self.assertTrue(destination.exists()) + self.assertEqual( + [(item.key, item.payload) for item in type(store)(destination).pull_all()], [("k1", b'{"n": 1}')] + ) + + def test_convert_never_writes_its_source(self) -> None: + for label, make in self._stores(): + for case, records in (("ok", {"k1": b'{"n": 1}'}), ("refused", {"k1": b'{"n": 1}', "k7": MIGRATION_RECORDS["k7"]})): + with self.subTest(store=label, case=case): + store = self._filled(make, f"ro-{label}-{case}", records) + before = (store.path.read_bytes(), store.path.stat().st_mtime_ns) + destination = self.directory / f"ro-{label}-{case}.out" + with contextlib.ExitStack() as stack: + for name in ("push", "push_all", "delete", "_replace_all"): + stack.enter_context(mock.patch.object(type(store), name, side_effect=AssertionError(name))) + try: + convert(store, destination, [define_document_type("tests.mig")]) + except CultCacheError: + self.assertEqual(case, "refused") + else: + self.assertEqual(case, "ok") + self.assertEqual((store.path.read_bytes(), store.path.stat().st_mtime_ns), before) + + def test_a_writer_after_the_read_keeps_its_record_and_convert_returns_normally(self) -> None: + for label, make in self._stores(): + with self.subTest(store=label): + store = self._filled(make, f"late-{label}", {"k1": b'{"n": 1}'}) + writer = type(store)(store.path) + real = type(store).pull_all + interleaved: list[bool] = [] + + def pull_all(self_store): + result = real(self_store) + if not interleaved: + interleaved.append(True) + writer.push(CultCacheEnvelope.create(key="w", type="tests.mig", payload=b'{"late": true}')) + return result + + destination = self.directory / f"late-{label}.out" + with mock.patch.object(type(store), "pull_all", pull_all): + count = convert(store, destination, [define_document_type("tests.mig")]) + self.assertEqual(count, 1) + self.assertEqual({item.key for item in store.pull_all()}, {"k1", "w"}) + self.assertEqual({item.key for item in type(store)(destination).pull_all()}, {"k1"}) + + def test_an_existing_destination_is_refused_before_the_source_is_read(self) -> None: + for label, make in self._stores(): + with self.subTest(store=label): + store = self._filled(make, f"taken-{label}", {"k1": b'{"n": 1}'}) + definitions = [define_document_type("tests.mig")] + earlier = self.directory / f"earlier-{label}.out" + convert(store, earlier, definitions) + empty = self.directory / f"empty-{label}.out" + empty.write_bytes(b"") + for destination in (store.path, earlier, empty): + before = destination.read_bytes() + source_before = store.path.read_bytes() + with mock.patch.object(type(store), "pull_all") as pull_all: + with self.assertRaises(CultCacheError) as caught: + convert(store, destination, definitions) + with self.assertRaises(CultCacheError) as again: + convert(store, destination, definitions) + self.assertEqual(str(caught.exception), DESTINATION_EXISTS) + self.assertEqual(str(again.exception), str(caught.exception)) + self.assertIsNone(caught.exception.__context__) + self.assertEqual(pull_all.call_count, 0) + self.assertEqual((destination.read_bytes(), store.path.read_bytes()), (before, source_before)) + + def test_a_refusal_leaves_no_destination_and_names_only_the_class(self) -> None: + for refusal_class in (AssertionError, TypeError, KeyError): + with self.subTest(refusal=refusal_class.__name__): + + def validate(value: object) -> None: + raise refusal_class(CANARY) + + store = self._filled( + lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc"), + f"validated-{refusal_class.__name__}", + {"k1": b'{"n": 1}'}, + ) + destination = self.directory / f"validated-{refusal_class.__name__}.out" + with self.assertRaises(CultCacheError) as caught: + convert(store, destination, [define_document_type("tests.mig", validate=validate)]) + self.assertEqual( + str(caught.exception), + f"Cannot convert tests.mig record k1: {refusal_class.__name__}; no store was written and the source is unchanged.", + ) + self.assertNotIn(CANARY, _chain_text(caught.exception)) + self.assertIsNone(caught.exception.__context__) + self.assertIsNone(caught.exception.__cause__) + self.assertFalse(destination.exists()) + + def test_an_integer_beyond_64_bits_is_refused_by_code_and_the_fixed_record_converts(self) -> None: + for label, make in self._stores(): + with self.subTest(store=label): + store = self._filled(make, f"big-{label}", MIGRATION_RECORDS) + destination = self.directory / f"big-{label}.out" + definition = define_document_type("tests.mig") + with self.assertRaises(CultCacheError) as caught: + convert(store, destination, [definition]) + for part in ("tests.mig", "k7", "integer_out_of_range"): + self.assertIn(part, str(caught.exception)) + self.assertNotIn(BIG_DIGITS[:8], _chain_text(caught.exception)) + self.assertIsNone(caught.exception.__context__) + self.assertFalse(destination.exists()) + store.push(CultCacheEnvelope.create(key="k7", type="tests.mig", payload=b'{"big": 1}')) + self.assertEqual(convert(store, destination, [definition]), 7) + self.assertEqual(len(self._pull_with(type(store)(destination), MESSAGEPACK_PAYLOAD_CODEC)), 7) + + def test_the_destination_is_fsynced_once_and_a_failed_sync_leaves_none(self) -> None: + store = self._filled( + lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc"), "sync", {"k1": b'{"n": 1}'} + ) + definitions = [define_document_type("tests.mig")] + destination = self.directory / "sync.out" + seen: list[int] = [] + real = os.fsync + + def fsync(descriptor: int) -> None: + seen.append(os.fstat(descriptor).st_size) + real(descriptor) + + with mock.patch("os.fsync", fsync): + convert(store, destination, definitions) + self.assertEqual(seen, [destination.stat().st_size]) + self.assertGreater(seen[0], 0) + failing = self.directory / "sync-failed.out" + with mock.patch("os.fsync", side_effect=OSError("disk")): + with self.assertRaises(OSError): + convert(store, failing, definitions) + self.assertFalse(failing.exists()) + + def test_the_json_lines_bytes_are_key_sorted_utf8_lines_ending_in_lf(self) -> None: + later = CultCacheEnvelope(key="é", type="b.t", payload=b"x", stored_at="T1") + earlier = CultCacheEnvelope(key="a", type="a.t", payload=b"yy", stored_at="T2") + expected = ( + '{"key": "a", "payload": "eXk=", "stored_at": "T2", "type": "a.t"}\n' + '{"key": "é", "payload": "eA==", "stored_at": "T1", "type": "b.t"}\n' + ).encode("utf-8") + self.assertEqual(JsonLinesBackingStore._file_bytes([later, earlier]), expected) + self.assertEqual(JsonLinesBackingStore._file_bytes([]), b"") + + @unittest.skipIf(os.name != "posix", "an open file cannot be removed by someone else on Windows") + def test_a_destination_removed_by_someone_else_does_not_mask_the_refusal(self) -> None: + store = self._filled( + lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc"), "vanished", {"k1": b'{"n": 1}'} + ) + destination = self.directory / "vanished.out" + real = SingleFileMessagePackBackingStore.pull_all + + def pull_all(self_store): + result = real(self_store) + destination.unlink() + return result + + def validate(value: object) -> None: + raise AssertionError(CANARY) + + with mock.patch.object(SingleFileMessagePackBackingStore, "pull_all", pull_all): + with self.assertRaises(CultCacheError) as caught: + convert(store, destination, [define_document_type("tests.mig", validate=validate)]) + self.assertIn("AssertionError", str(caught.exception)) + self.assertIsNone(caught.exception.__context__) + + def test_convert_finds_a_record_by_its_catalog_schema_name_and_skips_unlisted_types_anywhere(self) -> None: + store = SingleFileMessagePackBackingStore(self.directory / "names.cc") + store.push(CultCacheEnvelope.create(key="a", type="aaa.unlisted", payload=b"\x00opaque")) + store.push(CultCacheEnvelope.create(key="n", type="tests.by-name", payload=b'{"n": 1}')) + document = define_document_type("tests.by-type", schema_name="tests.by-name") + self.assertEqual(convert(store, self.directory / "names.out", [document]), 1) + converted = {item.key: item.payload for item in SingleFileMessagePackBackingStore(self.directory / "names.out").pull_all()} + self.assertEqual(converted["n"], MESSAGEPACK_PAYLOAD_CODEC.encode({"n": 1})) + self.assertEqual(converted["a"], b"\x00opaque") + + def test_convert_reads_under_the_source_codec_and_writes_under_the_definitions(self) -> None: + store = self._filled( + lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc"), + "codec", + {"k1": MESSAGEPACK_PAYLOAD_CODEC.encode({"n": 1})}, + ) + definition = define_document_type("tests.mig", payload_codec=JSON_PAYLOAD_CODEC) + destination = self.directory / "codec.out" + self.assertEqual(convert(store, destination, [definition], source_codec=MESSAGEPACK_PAYLOAD_CODEC), 1) + self.assertEqual(SingleFileMessagePackBackingStore(destination).pull_all()[0].payload, b'{"n": 1}') + + def test_convert_takes_only_existing_library_file_stores(self) -> None: + destination = self.directory / "foreign.out" + with self.assertRaises(TypeError) as caught: + convert(object(), destination, [define_document_type("tests.mig")]) + self.assertEqual(str(caught.exception), "convert takes a library file store, not object") + self.assertFalse(destination.exists()) + missing = SingleFileMessagePackBackingStore(self.directory / "missing.cc") + with self.assertRaises(CultCacheError) as caught: + convert(missing, destination, [define_document_type("tests.mig")]) + self.assertEqual(str(caught.exception), SOURCE_MISSING) + self.assertFalse(destination.exists()) + + def test_convert_refusals_name_no_path(self) -> None: + for label, make in self._stores(): + with self.subTest(store=label): + folder = self.directory / label / f"{CANARY}-dir" + folder.mkdir(parents=True) + definitions = [define_document_type("tests.mig")] + store = type(make("x"))(folder / f"{CANARY}-src") + taken = type(store)(folder / f"{CANARY}-taken") + taken.push(CultCacheEnvelope.create(key="k", type="tests.mig", payload=b'{"n": 1}')) + store.push(CultCacheEnvelope.create(key="k", type="tests.mig", payload=b'{"n": 1}')) + missing = type(store)(folder / f"{CANARY}-missing") + dangling = folder / f"{CANARY}-dangling" + os.symlink(folder / f"{CANARY}-nowhere", dangling) + raced = folder / f"{CANARY}-raced" + real_exists = Path.exists + + def exists_then_created(path): + if path == raced: + raced.write_bytes(b"raced") + return False + return real_exists(path) + + for expected, call in ( + (SOURCE_MISSING, lambda: convert(missing, folder / f"{CANARY}-out", definitions)), + (DESTINATION_EXISTS, lambda: convert(store, taken.path, definitions)), + # The late refusal: the create itself finds the destination (a dangling symlink, or a creator + # racing between the existence check and the create). + (DESTINATION_EXISTS, lambda: convert(store, dangling, definitions)), + (DESTINATION_EXISTS, lambda: self._convert_racing(exists_then_created, store, raced, definitions)), + ): + with self.assertRaises(CultCacheError) as caught: + call() + refusal = caught.exception + self.assertEqual(str(refusal), expected) + notes = repr(getattr(refusal, "__notes__", None)) + for text in (str(refusal), repr(refusal), repr(refusal.args), repr(vars(refusal)), notes, _chain_text(refusal)): + self.assertNotIn(CANARY, text) + self.assertNotIn(str(folder), text) + self.assertIsNone(refusal.__context__) + self.assertIsNone(refusal.__cause__) + + @staticmethod + def _convert_racing(exists, store, destination, definitions): + with mock.patch.object(Path, "exists", exists): + return convert(store, destination, definitions) + + def _one_record(self, label, name): + store = self._filled(dict(self._stores())[label], name, {"k1": b'{"n": 1}'}) + return store, self.directory / f"{name}.out", [define_document_type("tests.mig")] + + def test_convert_interrupt_leaves_no_destination(self) -> None: + import cultcache_py.cache as cache_module + + real_open = open + + class FirstWriteInterrupts: + def __init__(self, handle) -> None: + self.handle = handle + + def __enter__(self): + self.handle.__enter__() + return self + + def __exit__(self, *exc): + return self.handle.__exit__(*exc) + + def write(self, data): + raise KeyboardInterrupt + + def __getattr__(self, name): + return getattr(self.handle, name) + + def created_then_interrupted(path, mode): + real_open(path, mode).close() + raise KeyboardInterrupt + + def created_then_first_write_interrupted(path, mode): + return FirstWriteInterrupts(real_open(path, mode)) + + for label, _ in self._stores(): + for case, seam in ( + ("raises-after-create", created_then_interrupted), + ("first-write-raises", created_then_first_write_interrupted), + ): + with self.subTest(store=label, case=case): + store, destination, definitions = self._one_record(label, f"int-{label}-{case}") + before = store.path.read_bytes() + with mock.patch.object(cache_module, "open", seam, create=True): + with self.assertRaises(KeyboardInterrupt): + convert(store, destination, definitions) + self.assertFalse(destination.exists()) + self.assertEqual(store.path.read_bytes(), before) + self.assertEqual(convert(store, destination, definitions), 1) + self.assertTrue(destination.exists()) + + def test_convert_never_removes_a_destination_it_did_not_create(self) -> None: + import cultcache_py.cache as cache_module + + for label, _ in self._stores(): + with self.subTest(store=label): + store, destination, definitions = self._one_record(label, f"theirs-{label}") + destination.write_bytes(b"theirs") + with mock.patch.object(cache_module, "open", side_effect=KeyboardInterrupt, create=True): + with self.assertRaises(CultCacheError) as caught: + convert(store, destination, definitions) + self.assertEqual(str(caught.exception), DESTINATION_EXISTS) + self.assertEqual(destination.read_bytes(), b"theirs") + # Another process creates the destination between the existence check and the create. + other = self.directory / f"raced-{label}.out" + real_exists = Path.exists + + def exists(path): + if path == other: + other.write_bytes(b"raced") + return False + return real_exists(path) + + with mock.patch.object(Path, "exists", exists): + with self.assertRaises(CultCacheError) as caught: + convert(store, other, definitions) + self.assertEqual(str(caught.exception), DESTINATION_EXISTS) + self.assertEqual(other.read_bytes(), b"raced") + + def test_a_destination_under_a_missing_directory_is_created(self) -> None: + store = self._filled( + lambda name: SingleFileMessagePackBackingStore(self.directory / f"{name}.cc"), "nested", {"k1": b'{"n": 1}'} + ) + destination = self.directory / "a" / "b" / "nested.out" + self.assertEqual(convert(store, destination, [define_document_type("tests.mig")]), 1) + self.assertTrue(destination.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index 793933ff..4448029c 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -2,6 +2,7 @@ import collections import dataclasses +import json import shutil import tempfile import traceback @@ -15,8 +16,12 @@ from cultcache_py import ( MAX_PAYLOAD_DEPTH, MAX_SLOT_KEY, CultCache, + JSON_PAYLOAD_CODEC, + MESSAGEPACK_PAYLOAD_CODEC, + PayloadCodecFault, CultCachePayloadDepthError, DatabaseEntryField, + PayloadCodec, SlotCodec, SingleFileMessagePackBackingStore, UnionArm, @@ -595,17 +600,18 @@ class BoundsVectorTests(unittest.TestCase): with self.assertRaises(CultCachePayloadDepthError) as caught: tree.encode_payload(_nest(5000)) self.assertIn("refused on write", str(caught.exception)) - document = define_document_type("tests.json_tree") - with self.assertRaises(CultCachePayloadDepthError) as read: - document.decode_payload(b"[" * 300 + b"]" * 300) - self.assertIn("refused on read", str(read.exception)) + for codec, deep in ((None, b"\x91" * 300 + b"\xc0"), (JSON_PAYLOAD_CODEC, b"[" * 300 + b"]" * 300)): + document = define_document_type("tests.json_tree") if codec is None else define_document_type("tests.json_tree", payload_codec=codec) + with self.assertRaises(CultCachePayloadDepthError) as read: + document.decode_payload(deep) + self.assertIn("refused on read", str(read.exception)) def test_a_codec_that_exhausts_the_interpreter_stack_keeps_its_own_recursion_error_on_each_side(self) -> None: def boom(value: object) -> object: return boom(value) document = define_document_type("tests.boom", encode=boom, decode=boom) - for run in (lambda: document.encode_payload("x"), lambda: document.decode_payload(b'"x"')): + for run in (lambda: document.encode_payload("x"), lambda: document.decode_payload(b"\xa1x")): try: run() except RecursionError as caught: @@ -620,11 +626,11 @@ class BoundsVectorTests(unittest.TestCase): "tests.union_surface payload nests 257 containers deep; the limit is 256 (refused on read).", ) - def test_a_json_payload_past_the_interpreters_own_stack_keeps_the_decoders_recursion_error(self) -> None: - document = define_document_type("tests.json_tree") - with self.assertRaises(RecursionError) as caught: + def test_json_text_past_the_interpreters_own_stack_is_the_depth_refusal(self) -> None: + document = define_document_type("tests.json_tree", payload_codec=JSON_PAYLOAD_CODEC) + with self.assertRaises(CultCachePayloadDepthError) as caught: document.decode_payload(b"[" * 100000) - self.assertNotIsInstance(caught.exception, CultCachePayloadDepthError) + self.assertIn("refused on read", str(caught.exception)) def test_a_refusal_reports_the_depth_where_the_walk_stopped_whatever_the_payloads_real_depth(self) -> None: document = define_document_type("tests.json_tree", encode=lambda value: value) @@ -633,7 +639,7 @@ class BoundsVectorTests(unittest.TestCase): document.encode_payload(_nest(depth)) self.assertEqual((write.exception.depth, write.exception.limit), (257, 256), depth) with self.assertRaises(CultCachePayloadDepthError) as read: - document.decode_payload(b"[" * depth + b"]" * depth) + document.decode_payload(b"\x91" * depth + b"\xc0") self.assertEqual((read.exception.depth, read.exception.limit), (257, 256), depth) def test_the_validator_sees_the_value_on_write_and_the_decoded_value_on_read(self) -> None: @@ -650,6 +656,9 @@ class BoundsVectorTests(unittest.TestCase): document.encode_payload(loop) +_STUB_CODEC = PayloadCodec("test", lambda tree: b"ok", lambda payload: payload) + + def _nest(depth: int, mixed: bool = False) -> object: value: object = "leaf" for level in range(depth): @@ -738,18 +747,24 @@ class PayloadDepthRuleTests(unittest.TestCase): def test_the_limit_is_256_and_depth_256_is_accepted_on_read_and_write_while_257_is_refused(self) -> None: self.assertEqual(MAX_PAYLOAD_DEPTH, 256) - document = define_document_type("tests.json_tree", encode=lambda value: value) - for mixed in (False, True): - at_limit = _nest(256, mixed) - self.assertEqual(document.decode_payload(document.encode_payload(at_limit)), at_limit) - past = _nest(257, mixed) - with self.assertRaises(CultCachePayloadDepthError): - document.encode_payload(past) - with self.assertRaises(CultCachePayloadDepthError): - document.decode_payload(document.payload_encoder(past)) + for codec in (None, JSON_PAYLOAD_CODEC): + document = define_document_type("tests.json_tree", encode=lambda value: value, **({} if codec is None else {"payload_codec": codec})) + for mixed in (False, True): + at_limit = _nest(256, mixed) + self.assertEqual(document.decode_payload(document.encode_payload(at_limit)), at_limit) + past = _nest(257, mixed) + with self.assertRaises(CultCachePayloadDepthError): + document.encode_payload(past) + raw = json.dumps(past).encode() if codec is JSON_PAYLOAD_CODEC else msgpack.packb(past, use_bin_type=True) + with self.assertRaises(CultCachePayloadDepthError): + document.decode_payload(raw) + if codec is None: + with self.assertRaises(PayloadCodecFault) as refused: + MESSAGEPACK_PAYLOAD_CODEC.encode(past) + self.assertEqual(refused.exception.code, "depth_limit") def test_a_tuple_and_a_container_dict_key_count_as_containers(self) -> None: - document = define_document_type("tests.tuple_tree", encode=lambda value: value, payload_encoder=lambda tree: b"ok") + document = define_document_type("tests.tuple_tree", encode=lambda value: value, payload_codec=_STUB_CODEC) self.assertEqual(document.encode_payload(_as_tuple(_nest(256))), b"ok") with self.assertRaises(CultCachePayloadDepthError): document.encode_payload(_as_tuple(_nest(257))) @@ -765,19 +780,21 @@ class PayloadDepthRuleTests(unittest.TestCase): "userlist": lambda inner: collections.UserList([inner]), } for name, wrap in wrappers.items(): - document = define_document_type(f"tests.{name}", encode=lambda value: value, payload_encoder=lambda tree: b"ok") + document = define_document_type(f"tests.{name}", encode=lambda value: value, payload_codec=_STUB_CODEC) at_limit: object = "leaf" for _ in range(256): at_limit = wrap(at_limit) self.assertEqual(document.encode_payload(at_limit), b"ok", name) with self.assertRaises(CultCachePayloadDepthError, msg=name): document.encode_payload(wrap(at_limit)) - decoding = define_document_type(f"tests.{name}.read", payload_decoder=lambda payload, w=wrap: w(at_limit)) + decoding = define_document_type( + f"tests.{name}.read", payload_codec=PayloadCodec("test", lambda tree: b"ok", lambda payload, w=wrap: w(at_limit)) + ) with self.assertRaises(CultCachePayloadDepthError, msg=name): decoding.decode_payload(b"x") def test_text_and_bytes_are_leaves_however_long(self) -> None: - document = define_document_type("tests.leaves", encode=lambda value: value, payload_encoder=lambda tree: b"ok") + document = define_document_type("tests.leaves", encode=lambda value: value, payload_codec=_STUB_CODEC) self.assertEqual(document.encode_payload([["x" * 100000, b"y" * 100000, bytearray(5), memoryview(b"z")]]), b"ok") def test_a_scalar_payload_has_no_depth(self) -> None: diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index 23e217e8..af5bbd9e 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -13,6 +13,7 @@ only in git history. ### Changed +- `CultCache::delete::` of a key nothing holds now refuses, as `put` and `is_writable` do, when `T` has no home store ("No backing store is registered…") or when a store carries a foreign record at the key or another registered type holds it (before: `Ok(false)`). - `is_stored_at_stamp(&str)` (new, public) is the one shape a peer's `storedAt` must parse as, pinned by `tests/vectors/stored-at-stamp` (RFC 3339 `YYYY-MM-DDThh:mm:ss`, a 1 to 9 digit fraction, `Z` or `+hh:mm`/`-hh:mm`; uppercase `T`/`Z`, a real date, seconds 00-59, years 0001-9999), hand-checked because `chrono`'s RFC 3339 reader takes other shapes. `put_raw_envelope` and `put_prepared_batch` refuse a `stored_at` that is not a stamp (before: only a blank one) and mint nothing (variants:ruling:raw-put-without-stamp-refused). - The single-file store's `push`, `push_all` and `commit_if` stamp a replacing write strictly later than the record it replaces at the same key (a staged stamp stands only when it parses and is strictly later; otherwise the stamp is minted as the replaced stamp plus one microsecond, or now if later), with no identical-content exception, as C# does. A stamp with no successor (the replaced stamp is at year 9999's last microsecond) refuses the write and writes nothing, as C# `MintStoredAt` does. `commit_if` replaces only the record of the entry's own type under the key, so a retype is refused as `push` refuses it. The cache holds the stamp the store wrote. The redb stores and the store-level exact-envelope writes (`compare_and_swap_entry` and its family) still write the caller's envelope unchanged. - The single-file store re-checks for a link once the store lock is held (every write and the try-exchange), so a link put at the path while a writer waited on the lock is refused with `StoreLinkedError` and the lock released, as C# does, instead of being replaced. Pinned by a test that puts the link while a second holder owns the lock. diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 0e60a0c2..614f7861 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -3487,6 +3487,8 @@ impl CultCache { /// The one decode behind every write entry and [`CultCache::is_writable`]: the held envelope at `(type_id, key)` /// read under the registered type's mark. + /// Each write over a held key pays one full typed decode of the held payload (about 40% of a `put` of a 200,000-element + /// record, measured in release); this cost is accepted deliberately, with no cached mark, so the check keeps no second state. fn held_finding(&self, type_id: &str, key: &str) -> Result { let Some(definition) = self.definitions.get(type_id) else { bail!("No entry type registered for CultCache envelope type {type_id:?}"); @@ -3985,15 +3987,22 @@ impl CultCache { self.require_loaded()?; self.require_entry_type::()?; self.refuse_unread_slots(T::TYPE, key)?; + let home = self.home_index(T::TYPE)?; let id = entry_id_parts(T::TYPE, key); let Some(entry) = self.entries.get(&id).cloned() else { - if let Some((_, foreign)) = self.foreign.iter().find(|(_, foreign)| foreign.key == key) { - return Err(foreign_conflict(foreign, T::TYPE)); - } + // The retype judge a write runs: a foreign record or another registered type holding the key refuses the removal. + let probe = CultCacheEnvelope { + key: key.to_string(), + r#type: T::TYPE.to_string(), + payload: Vec::new(), + stored_at: String::new(), + schema_id: None, + }; + self.refuse_retype(&self.entries, &self.foreign, &probe)?; return Ok(false); }; let released = self.projected_slots(&entry)?; - if let Some(home) = self.home_index(T::TYPE)? { + if let Some(home) = home { self.stores[home].store.delete(&entry)?; } self.entries.remove(&id); diff --git a/packages/cultcache-rs/tests/slot_readonly.rs b/packages/cultcache-rs/tests/slot_readonly.rs index 7851c818..1fb0909b 100644 --- a/packages/cultcache-rs/tests/slot_readonly.rs +++ b/packages/cultcache-rs/tests/slot_readonly.rs @@ -477,6 +477,21 @@ fn is_writable_is_false_for_a_type_with_no_home_store() { assert!(cache.is_writable(SlotUnit::TYPE, "k").unwrap()); } +// For a homeless type, is_writable, put and delete of an unheld key agree: removal is refused like the write, not answered false. +#[test] +fn delete_of_an_unheld_key_of_a_type_with_no_home_store_is_refused_like_the_put() { + let dir = tempfile::tempdir().unwrap(); + let mut cache = registered(); + cache.add_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc")), [SlotUnit::TYPE]).unwrap(); + cache.pull_all_backing_stores().unwrap(); + assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap()); + let error = cache.delete::("k").unwrap_err(); + assert!(error.to_string().contains("No backing store is registered"), "{error:#}"); + // A homed type's unheld delete is still a plain false. + assert!(cache.is_writable(SlotUnit::TYPE, "k").unwrap()); + assert!(!cache.delete::("k").unwrap()); +} + #[test] fn load_soa_refuses_a_same_length_replacement_of_a_marked_record() { use cultcache_rs::CultCacheSoaTable; @@ -539,3 +554,53 @@ fn a_store_with_marked_records_and_a_tail_is_refused_and_its_cache_writes_nothin } assert_eq!(std::fs::read(&path).unwrap(), before); } + +// Attaching a store to a cache whose load was refused must keep the refusal: the write refusal still names why the load +// failed, not merely that the cache has not pulled. (Only a pulled cache falls back to unpulled when a store is attached.) +#[test] +fn a_store_attached_after_a_refused_load_keeps_the_refusal_reason() { + use cultcache_rs::CacheBackingStore; + use cultcache_rs::CultCacheLoadRefused; + let (_held, dir, path) = held_marked_and_clean(); + let mut tailed = vector("union-surface.msgpack"); + tailed.push(0xc0); + SingleFileMessagePackBackingStore::new(&path).push(&envelope("tailed", tailed)).unwrap(); + let mut cache = registered(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + assert!(cache.pull_all_backing_stores().is_err()); + cache.add_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("routed.cc")), [SlotUnit::TYPE]).unwrap(); + let error = cache.put("new", &constructed()).unwrap_err(); + let refused = error.downcast_ref::().unwrap_or_else(|| panic!("{error:#}")); + assert!(refused.reason.contains("bytes after"), "refusal reason lost: {}", refused.reason); +} + +// A key held by another registered type, or carried as a foreign record: put, is_writable and typed delete all refuse, and +// delete refuses with the put's conflict (delete used to answer Ok(false)). This agreement is for those two cases; for a +// type with no home store delete's home-store error comes before the retype judge, and both still refuse. +#[test] +fn delete_of_a_key_held_by_another_registered_type_is_refused_like_the_put() { + let (mut cache, _dir, _path) = held_marked_and_clean(); + assert!(!cache.is_writable(SlotUnit::TYPE, "clean").unwrap()); + let put = cache.put("clean", &SlotUnit { name: "n".into(), outcome: UnitOutcome::Denied }).unwrap_err(); + let delete = cache.delete::("clean").unwrap_err(); + let (put, delete) = (conflict(&put), conflict(&delete)); + assert_eq!(delete.record_key, "clean"); + assert_eq!(delete.record_key, put.record_key); + assert_eq!(delete.schema_names, put.schema_names); + assert!(delete.schema_names.contains(&UnionSurface::TYPE.to_string()), "{:?}", delete.schema_names); + assert!(cache.snapshot_envelopes().iter().any(|held| held.key == "clean" && held.r#type == UnionSurface::TYPE)); + + // A foreign record at the key: only UnionSurface registered, the store's unit-arm record is foreign here. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("extra-slot-store.cc"); + std::fs::write(&path, without_variant_row(&vector("extra-slot-store.cc"))).unwrap(); + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); + assert!(!cache.is_writable(UnionSurface::TYPE, "unit-arm").unwrap()); + let put = cache.put("unit-arm", &constructed()).unwrap_err(); + let delete = cache.delete::("unit-arm").unwrap_err(); + assert_eq!(conflict(&delete).record_key, "unit-arm"); + assert_eq!(conflict(&delete).schema_names, conflict(&put).schema_names); +} diff --git a/packages/cultnet-py/src/cultnet_py/interop_peer.py b/packages/cultnet-py/src/cultnet_py/interop_peer.py index 56dddec8..9c2c601a 100644 --- a/packages/cultnet-py/src/cultnet_py/interop_peer.py +++ b/packages/cultnet-py/src/cultnet_py/interop_peer.py @@ -963,8 +963,6 @@ def binding( document_type, encode=encode_slots, decode=decode_slots, - payload_encoder=lambda value: msgpack.packb(value, use_bin_type=True), - payload_decoder=lambda payload: msgpack.unpackb(payload, raw=False), schema_id=schema_id, schema_name=document_type, schema_version=schema_version, diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 41d63295..a112a785 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any, Iterable from cultcache_py import CultCache, CultCacheEnvelope -from cultcache_py.documents import DocumentDefinition +from cultcache_py.documents import MESSAGEPACK_PAYLOAD_CODEC, DocumentDefinition from cultcache_py.stores import parse_stored_at from .shard_log import CultNetShardLogResponse @@ -190,9 +190,12 @@ def resolve_document_for_raw_record( def _infer_schema_version_from_payload(payload: bytes) -> str | None: + """Routing only, as the C# `CultNetDocumentRegistry.TryResolveDescriptorByPayloadSchema` does: reads a schema stamp + off a raw record whose schema id is unknown, to pick a definition. It decides no format; the definition it picks + has a codec that decodes the payload or refuses it. The codec's reader reads the stamp, so the router decides no key.""" import msgpack # type: ignore try: - decoded = msgpack.unpackb(payload, raw=False) + decoded = MESSAGEPACK_PAYLOAD_CODEC.decode(payload) except msgpack.ExtraData as extra: # The hint is read from the first value; the tail is not this function's to judge. The routed definition refuses # it by count when the record is applied, as it does for a record routed by a registered schema id. Treating a diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 73b41a79..55622aea 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -25,6 +25,7 @@ from cultnet_py.compare_csharp import ( ) from cultnet_py.interop_peer import ( append_shard_log_put, + binding, build_state, handle_server_message, raw_snapshot_response, @@ -1976,8 +1977,7 @@ class CultNetTests(unittest.TestCase): from cultcache_py import define_document_type from cultnet_py import resolve_document_for_raw_record - document = define_document_type("tests.tail_note", schema_id="tests.tail_note", schema_version="tests.tail_note.v1", - payload_encoder=msgpack.packb, payload_decoder=lambda payload: msgpack.unpackb(payload, raw=False)) + document = define_document_type("tests.tail_note", schema_id="tests.tail_note", schema_version="tests.tail_note.v1") known = document.catalog_entry().schema_id head = msgpack.packb(["tests.tail_note.v1", "x"], use_bin_type=True) @@ -2119,6 +2119,42 @@ class CultNetTests(unittest.TestCase): self.assertEqual([change.change_kind for change in typed_applied_log], ["updated", "removed"]) self.assertIsNone(cache.get(document, "item:1")) + def test_the_stamp_router_judges_no_map_key_and_the_codec_alone_does(self) -> None: + import msgpack # type: ignore + from cultnet_py import resolve_document_for_raw_record + + document = define_database_entry_type( + "replica.stamped", + [("schema_version", 0), ("value", 1)], + schema_id="replica.stamped.current", + schema_name="replica.stamped", + schema_version="replica.stamped.v1", + ) + by_schema_id = {"replica.stamped.current": document, "replica.stamped": document} + + def routed(tree: object) -> object: + record = {"payload": msgpack.packb(tree, use_bin_type=True)} + return resolve_document_for_raw_record(by_schema_id, "sha256:unknown", record) + + for name, tree in ( + ("array", ["replica.stamped.v1", {1: "a"}]), + ("map", {"schemaVersion": "replica.stamped.v1", "v": {1: "a"}}), + ("nested", ["replica.stamped.v1", {"a": {2: [{3: 4}]}}]), + ("snake", {"schema_version": "replica.stamped.v1", "v": {1: "a"}}), + ): + with self.subTest(form=name): + self.assertIs(routed(tree), document) + for name, tree in ( + ("array", ["replica.stamped.v1", {None: "a"}]), + ("map", {"schemaVersion": "replica.stamped.v1", "v": {None: "a"}}), + ("not-a-string", {"schemaVersion": 5, "v": {1: "a"}}), + ("unknown-name", ["replica.other.v1", {1: "a"}]), + ): + with self.subTest(form=name): + with self.assertRaises(KeyError) as caught: + routed(tree) + self.assertEqual(caught.exception.args, ("sha256:unknown",)) + def test_cultnet_replication_helpers_recover_schema_stamped_raw_records(self) -> None: document = define_database_entry_type( "replica.runtime-policy", @@ -2678,6 +2714,12 @@ class CultNetTests(unittest.TestCase): self.assertEqual(response["shardLogSequence"], 1) self.assertEqual([record["recordKey"] for record in response["documents"]], ["note:logged"]) + def test_python_interop_peer_binding_writes_plain_messagepack_slots(self) -> None: + # The bytes the binding wrote before it stopped passing its own codec lambdas: packb of the slot array. + note = binding("tests.note", "tests.note.v1", "tests.note.v1", "{}", lambda value: ["a", 1], lambda slots: slots) + self.assertEqual(note.document.encode_payload({}), bytes.fromhex("92a16101")) + self.assertEqual(note.document.decode_payload(bytes.fromhex("92a16101")), ["a", 1]) + def test_python_interop_peer_refuses_v1_selection_messages(self) -> None: # R-2, owed by Cut 1 (docs/cultnet-selection-cut.md, section 9): a v0-only peer refuses any # non-v0 message loudly - cultnet.error.v0 { code: "unsupported_schema_version" } - rather than