diff --git a/.gitattributes b/.gitattributes index a499574a..086f17af 100644 --- a/.gitattributes +++ b/.gitattributes @@ -38,3 +38,6 @@ packages/cultmesh-ts/native/MANIFEST.txt text eol=lf # LF-configured or Linux clone builds the same bytes as a Windows one. The index # stays LF; only the checkout changes. packages/cultmath/src/** text eol=crlf + +# Stores the Python typed-leaf tests write for the C# reference to open. +tests/vectors/**/*.cc binary diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 23effdbd..41c3095c 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -67,6 +67,7 @@ store. - `define_document_type(...)` - `define_database_entry_type(...)` - `database_entry_field(...)` +- `leaves` (typed slot leaves) - `define_document_registry(...)` - `CultCache.builder()` - `register_document_type(...)` @@ -148,6 +149,55 @@ That matches the Rust `#[derive(DatabaseEntry)]` formatter shape and the C# `[Key(n)]` intent: schema evolution comes from stable field slots, not source member order. +## Typed Leaves + +A slot with no codec is untyped: it is an `object` in C#, cataloged `System.Object`, and takes any MessagePack value. +Give a slot a leaf from `cultcache_py.leaves` and it writes and reads exactly what the C# type does, refuses the rest +on write and on read, and carries the C# type name in the schema catalog, which the C# reference checks against its +own class. + +```python +from cultcache_py import database_entry_field, define_database_entry_type, leaves as t + +settings_doc = define_database_entry_type( + "settings", + [ + database_entry_field("theme", 0, codec=t.string), + database_entry_field("retries", 1, codec=t.int32, default=0), + database_entry_field("tags", 2, codec=t.list(t.string), default=list), + ], +) +``` + +| Leaf | C# type | Python value | +| --- | --- | --- | +| `t.string` | `string` | `str` | +| `t.boolean` | `bool` | `bool` | +| `t.bytes` | `byte[]` | `bytes`, `bytearray`, `memoryview`; a MessagePack str reads as its UTF-8 bytes | +| `t.int32`, `t.int64` | `int`, `long` | `int` within range (not `bool`, not `float`) | +| `t.float32`, `t.float64` | `float`, `double` | `int` or `float`; a float32 is written as MessagePack float 32 | +| `t.list(x)` | `List` | `list` or `tuple`, each element through `x` | +| `t.array(x)` | `T[]` | the same wire as `t.list(x)` | +| `t.any` | `object` | any plain MessagePack value or a timestamp; the default of an untyped slot | +| `t.nullable(x)` | `T?` | `None`, or whatever `x` takes | +| `t.lazy(get)` | | a leaf resolved on first use, for a union that refers to itself | + +Nil is refused unless the slot is wrapped in `t.nullable`. Wrap a C# `string`, `byte[]`, `List`, array, class or +interface slot that may hold null (`t.nullable(t.string)`), and a value type that may hold null (`int?` is +`t.nullable(t.int32)`, cataloged `System.Nullable`); a reference type keeps its own name. A `default` +fills a slot the payload is too short to have, never a nil that is present. + +A record or union is named for C# with `type_name=`, the class's `FullName` (a nested class joins with `+`): + +```python +inner = define_record_codec("Inner", [database_entry_field("n", 0, codec=t.string)], type_name="My.App.Doc+Inner") +shape = define_union("Shape", {0: ("Circle", circle)}, type_name="My.App.Doc+IShape") +``` + +Without `type_name=` the slot is cataloged `System.Object`. A refusal reads ` slot (): expected +, found `, prefixed `element : ` inside a list and `Union '' arm : ` inside a union, and never +carries the value. + ## Wire Parity See [docs/python-runtime-parity.md](../../docs/python-runtime-parity.md) for diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index 91656c35..dbfac403 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -1,5 +1,6 @@ from .backing_store import BackingStore, CultCacheEnvelope from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember +from . import leaves from .cache import CultCache, CultCacheBuilder from .documents import ( MAX_PAYLOAD_DEPTH, @@ -7,7 +8,6 @@ from .documents import ( CultCachePayloadDepthError, DatabaseEntryField, DocumentDefinition, - SlotCodec, UnionArm, database_entry_field, define_database_entry_type, @@ -16,6 +16,7 @@ from .documents import ( define_record_codec, define_union, ) +from .leaves import SlotCodec from .stores import JsonLinesBackingStore, SingleFileMessagePackBackingStore __all__ = [ @@ -34,6 +35,7 @@ __all__ = [ "SingleFileMessagePackBackingStore", "SlotCodec", "UnionArm", + "leaves", "database_entry_field", "define_database_entry_type", "define_document_registry", diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 293625ce..91d09ab8 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -7,6 +7,8 @@ from itertools import chain from typing import Any, Callable, Generic, Iterable, TypeVar from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember +from .leaves import Float32, SlotCodec +from .leaves import any as _any_leaf T = TypeVar("T") @@ -83,7 +85,7 @@ def _json_payload_decoder(payload: bytes) -> Any: return json.loads(payload.decode("utf-8")) -def _messagepack_payload_encoder(value: Any) -> bytes: +def _msgpack() -> Any: try: import msgpack # type: ignore except ModuleNotFoundError as exc: @@ -91,7 +93,35 @@ def _messagepack_payload_encoder(value: Any) -> bytes: "MessagePack payload encoding requires the optional 'msgpack' dependency. " "Install with: python -m pip install cultcache-py[msgpack]" ) from exc - return msgpack.packb(value, use_bin_type=True) + return msgpack + + +def _pack_tree(tree: Any) -> bytes: + """The bytes of the tree the leaves produced. `packb` writes everything but a `Float32`, which it refuses with a + TypeError; only then does an iterative walk pack each array and map header and each value, a `Float32` through a + single-float packer.""" + msgpack = _msgpack() + try: + return msgpack.packb(tree, use_bin_type=True) # type: ignore[no-any-return] + except TypeError: + pass + plain = msgpack.Packer(use_bin_type=True) + single = msgpack.Packer(use_bin_type=True, use_single_float=True) + out: list[bytes] = [] + stack = [tree] + while stack: + node = stack.pop() + if isinstance(node, Float32): + out.append(single.pack(node.value)) + elif isinstance(node, (list, tuple)): + out.append(plain.pack_array_header(len(node))) + stack.extend(reversed(node)) + elif isinstance(node, dict): + out.append(plain.pack_map_header(len(node))) + stack.extend(reversed([part for pair in node.items() for part in pair])) + else: + out.append(plain.pack(node)) + return b"".join(out) def _messagepack_payload_decoder(payload: bytes) -> Any: @@ -105,25 +135,18 @@ def _messagepack_payload_decoder(payload: bytes) -> Any: return msgpack.unpackb(payload, raw=False) -@dataclass(frozen=True) -class SlotCodec: - """Encodes and decodes the value of one slot: a nested record or a union.""" - - encode: Callable[[Any], Any] - decode: Callable[[Any], Any] - - @dataclass(frozen=True) class DatabaseEntryField: name: str key: int default: Any = MISSING - codec: SlotCodec | Callable[[], SlotCodec] | None = None - many: bool = False + codec: SlotCodec = _any_leaf def __post_init__(self) -> None: if not _is_key(self.key) or not 0 <= self.key <= MAX_SLOT_KEY: raise ValueError(f"DatabaseEntry field keys must be integers within 0..{MAX_SLOT_KEY}") + if not isinstance(self.codec, SlotCodec): + raise ValueError("DatabaseEntry field codec must be a SlotCodec: a leaf from cultcache_py.leaves, or leaves.lazy") def database_entry_field( @@ -131,14 +154,11 @@ def database_entry_field( key: int, *, default: Any = MISSING, - codec: SlotCodec | Callable[[], SlotCodec] | None = None, - many: bool = False, + codec: SlotCodec = _any_leaf, ) -> DatabaseEntryField: - """A slot. `codec` makes the slot a nested record or union (a zero-argument callable - returning the codec lets a union refer to itself); `many` makes it a list of them.""" - if many and codec is None: - raise ValueError("A many DatabaseEntry field requires a codec") - return DatabaseEntryField(name=name, key=key, default=default, codec=codec, many=many) + """A slot. `codec` is the slot's leaf (cultcache_py.leaves, a record or a union); with none the slot is untyped, + an `object` in C#. A `default` fills a slot the payload is too short to have, never a nil.""" + return DatabaseEntryField(name=name, key=key, default=default, codec=codec) @dataclass(frozen=True) @@ -166,7 +186,11 @@ class DocumentDefinition(Generic[T]): content_hash: str | None = None canonical_schema_json: str | None = None compatible_schema_ids: tuple[str, ...] = field(default_factory=tuple) - members: tuple[CultCacheSchemaCatalogMember, ...] = field(default_factory=tuple) + members: tuple[CultCacheSchemaCatalogMember, ...] | Callable[[], tuple[CultCacheSchemaCatalogMember, ...]] = field( + default_factory=tuple + ) + """The catalog rows, or a function returning them: a document made of leaves derives them when the catalog is first + built, so a `lazy` leaf can name a definition made later.""" _catalog_entry: CultCacheSchemaCatalogEntry | None = field(default=None, init=False, repr=False, compare=False) def encode_payload(self, value: T) -> bytes: @@ -190,6 +214,7 @@ class DocumentDefinition(Generic[T]): def catalog_entry(self) -> CultCacheSchemaCatalogEntry: if self._catalog_entry is not None: return self._catalog_entry + members = tuple(self.members() if callable(self.members) else self.members) schema_id = self.schema_id or self.type schema_name = self.schema_name or self.type schema_version = self.schema_version or f"{schema_name}.v1" @@ -208,7 +233,7 @@ class DocumentDefinition(Generic[T]): "indexAlias": member.index_alias, "isName": member.is_name, } - for member in self.members + for member in members ], }, ensure_ascii=False, @@ -222,7 +247,7 @@ class DocumentDefinition(Generic[T]): content_hash=self.content_hash or schema_id, canonical_schema_json=canonical_schema_json, compatible_schema_ids=tuple(compatible_schema_ids), - members=tuple(self.members), + members=members, ) object.__setattr__(self, "_catalog_entry", catalog_entry) return catalog_entry @@ -245,7 +270,7 @@ def define_document_type( content_hash: str | None = None, canonical_schema_json: str | None = None, compatible_schema_ids: Iterable[str] | None = None, - members: Iterable[CultCacheSchemaCatalogMember] | None = None, + members: Iterable[CultCacheSchemaCatalogMember] | Callable[[], tuple[CultCacheSchemaCatalogMember, ...]] | None = None, ) -> DocumentDefinition[T]: return DocumentDefinition( type=type, @@ -263,7 +288,7 @@ def define_document_type( content_hash=content_hash, canonical_schema_json=canonical_schema_json, compatible_schema_ids=tuple(compatible_schema_ids or ()), - members=tuple(members or ()), + members=members if callable(members) else tuple(members or ()), ) @@ -293,19 +318,21 @@ def define_database_entry_type( normalized = _normalize_fields(type, fields) member_name_lookup = name if isinstance(name, str) else None index_lookup = indexes or {} - members = tuple( - CultCacheSchemaCatalogMember( - slot=field.key, - member_name=_pascal_case(field.name), - type_name="System.Object", - is_name=field.name == member_name_lookup, - index_alias=next( - (alias for alias, extractor in index_lookup.items() if extractor == field.name), - None, - ), + + def members() -> tuple[CultCacheSchemaCatalogMember, ...]: + return tuple( + CultCacheSchemaCatalogMember( + slot=field.key, + member_name=_pascal_case(field.name), + type_name=field.codec.type_name or _any_leaf.type_name, + is_name=field.name == member_name_lookup, + index_alias=next( + (alias for alias, extractor in index_lookup.items() if extractor == field.name), + None, + ), + ) + for field in normalized ) - for field in normalized - ) record = _record_codec(type, normalized, cls) @@ -317,7 +344,7 @@ def define_database_entry_type( global_document=global_document, name=name, indexes=indexes, - payload_encoder=_messagepack_payload_encoder, + payload_encoder=_pack_tree, payload_decoder=_messagepack_payload_decoder, schema_id=schema_id, schema_name=schema_name, @@ -334,17 +361,22 @@ def define_record_codec( fields: Iterable[DatabaseEntryField | tuple[str, int] | tuple[str, int, Any]], *, cls: type[Any] | None = None, + type_name: str | None = None, ) -> SlotCodec: - """A nested record: the same slot array as a document payload, usable as a field's codec or a union arm.""" - return _record_codec(name, _normalize_fields(name, fields), cls) + """A nested record: the same slot array as a document payload, usable as a field's codec or a union arm. + `type_name` is the C# FullName of the class it mirrors, for the catalog; without it the catalog names the slot as an untyped one.""" + return _record_codec(name, _normalize_fields(name, fields), cls, type_name) def define_union( name: str, - arms: Mapping[int, tuple[str, SlotCodec | Callable[[], SlotCodec] | None]], + arms: Mapping[int, tuple[str, SlotCodec | None]], + *, + type_name: str | None = None, ) -> SlotCodec: - """A union: nil, or `[armKey, [armSlots...]]`. Each arm is `(arm name, record codec)`; a unit arm has - no codec and is written `[armKey, []]`. A key the union does not declare is refused, never carried.""" + """A union: `[armKey, [armSlots...]]`. Each arm is `(arm name, record codec)`; a unit arm has no codec and is + written `[armKey, []]`. A key the union does not declare is refused, never carried, and so is nil: wrap the union + in `leaves.nullable` for a slot that may be null. `type_name` is the C# FullName of the interface, for the catalog.""" by_key = dict(arms) by_name = {arm_name: (key, codec) for key, (arm_name, codec) in by_key.items()} if not by_key: @@ -353,10 +385,12 @@ def define_union( raise ValueError(f"Union {name!r} has duplicate arm names") if any(not _is_key(key) or not _INT32_MIN <= key <= _INT32_MAX for key in by_key): raise ValueError(f"Union {name!r} arm keys must be integers within Int32") + if any(codec is not None and not isinstance(codec, SlotCodec) for _, codec in by_key.values()): + raise ValueError(f"Union {name!r} arm codecs must be SlotCodecs or None") def encode(value: Any) -> Any: if value is None: - return None + raise ValueError(f"Union {name!r}: expected an arm, found nil") if not isinstance(value, UnionArm) or value.name not in by_name: raise ValueError(f"Union {name!r} was given a value that is not one of its arms") key, codec = by_name[value.name] @@ -364,11 +398,14 @@ def define_union( if value.value is not None: raise ValueError(f"Union {name!r} arm {key} is a unit arm and carries no value") return [key, []] - return [key, _resolve(codec).encode(value.value)] + try: + return [key, codec.encode(value.value)] + except ValueError as error: + raise ValueError(f"Union {name!r} arm {key}: {error}") from None def decode(raw: Any) -> Any: if raw is None: - return None + raise ValueError(f"Union {name!r}: expected an arm, found nil") if not isinstance(raw, list) or len(raw) != 2: raise ValueError(f"Union {name!r} value is malformed: expected [armKey, [armSlots]]") key = raw[0] @@ -384,9 +421,14 @@ def define_union( if not isinstance(raw[1], list): raise ValueError(f"Union {name!r} value is malformed: expected [armKey, [armSlots]]") arm_name, codec = by_key[key] - return UnionArm(arm_name, None if codec is None else _resolve(codec).decode(raw[1])) + if codec is None: + return UnionArm(arm_name, None) + try: + return UnionArm(arm_name, codec.decode(raw[1])) + except ValueError as error: + raise ValueError(f"Union {name!r} arm {key}: {error}") from None - return SlotCodec(encode=encode, decode=decode) + return SlotCodec(encode=encode, decode=decode, type_name=type_name) def define_document_registry(*documents: DocumentDefinition[Any]) -> tuple[DocumentDefinition[Any], ...]: @@ -409,10 +451,6 @@ def _is_key(value: Any) -> bool: return isinstance(value, int) and not isinstance(value, bool) -def _resolve(codec: SlotCodec | Callable[[], SlotCodec]) -> SlotCodec: - return codec if isinstance(codec, SlotCodec) else codec() - - def _normalize_fields( type: str, fields: Iterable[DatabaseEntryField | tuple[str, int] | tuple[str, int, Any]], @@ -426,31 +464,20 @@ def _normalize_fields( return sorted(normalized, key=lambda field: field.key) -def _record_codec(type: str, normalized: list[DatabaseEntryField], cls: type[Any] | None) -> SlotCodec: +def _record_codec( + type: str, normalized: list[DatabaseEntryField], cls: type[Any] | None, type_name: str | None = None +) -> SlotCodec: max_key = normalized[-1].key - def encode_slot(field: DatabaseEntryField, value: Any) -> Any: - if field.codec is None or value is None: - return value - codec = _resolve(field.codec) - if field.many: - return [None if item is None else codec.encode(item) for item in value] - return codec.encode(value) - - def decode_slot(field: DatabaseEntryField, value: Any) -> Any: - if field.codec is None or value is None: - return value - codec = _resolve(field.codec) - if field.many: - if not isinstance(value, list): - raise ValueError(f"DatabaseEntry document type {type!r} slot {field.key} expected a MessagePack array") - return [None if item is None else codec.decode(item) for item in value] - return codec.decode(value) - def encode(value: Any) -> list[Any]: + if value is None: + raise ValueError(f"DatabaseEntry document type {type!r}: expected a record, found nil") slots = [None] * (max_key + 1) for field in normalized: - slots[field.key] = encode_slot(field, extract_value(value, field.name)) + try: + slots[field.key] = field.codec.encode(extract_value(value, field.name)) + except ValueError as error: + raise ValueError(f"{type} slot {field.key} ({field.name}): {error}") from None return slots def decode(raw: Any) -> Any: @@ -458,15 +485,20 @@ def _record_codec(type: str, normalized: list[DatabaseEntryField], cls: type[Any raise ValueError(f"DatabaseEntry document type {type!r} expected a MessagePack array") values: dict[str, Any] = {} for field in normalized: - value = decode_slot(field, raw[field.key]) if field.key < len(raw) else None - if value is None and field.default is not MISSING: - value = field.default() if callable(field.default) else field.default - values[field.name] = value + if field.key < len(raw): + try: + values[field.name] = field.codec.decode(raw[field.key]) + except ValueError as error: + raise ValueError(f"{type} slot {field.key} ({field.name}): {error}") from None + elif field.default is MISSING: + values[field.name] = None + else: + values[field.name] = field.default() if callable(field.default) else field.default if cls is None: return values return cls(**values) - return SlotCodec(encode=encode, decode=decode) + return SlotCodec(encode=encode, decode=decode, type_name=type_name) def _normalize_database_entry_field( diff --git a/packages/cultcache-py/src/cultcache_py/interop.py b/packages/cultcache-py/src/cultcache_py/interop.py index 021969f1..70521963 100644 --- a/packages/cultcache-py/src/cultcache_py/interop.py +++ b/packages/cultcache-py/src/cultcache_py/interop.py @@ -1,50 +1,29 @@ -from __future__ import annotations +from __future__ import annotations import argparse import json from typing import Any -import msgpack - -from . import CultCache, CultCacheSchemaCatalogMember, SingleFileMessagePackBackingStore, define_document_type +from . import CultCache, SingleFileMessagePackBackingStore, database_entry_field, define_database_entry_type, leaves as t INTEROP_SCHEMA_VERSION = "cultcache.interop_note.v1" -interop_note_document = define_document_type( +interop_note_document = define_database_entry_type( "cultcache.interop-note", - encode=lambda note: [ - note["schemaVersion"], - note["documentId"], - note["authorRuntimeId"], - note["title"], - note["body"], - note["tags"], + [ + database_entry_field("schemaVersion", 0, codec=t.nullable(t.string)), + database_entry_field("documentId", 1, codec=t.nullable(t.string)), + database_entry_field("authorRuntimeId", 2, codec=t.nullable(t.string)), + database_entry_field("title", 3, codec=t.nullable(t.string)), + database_entry_field("body", 4, codec=t.nullable(t.string)), + database_entry_field("tags", 5, default=list, codec=t.nullable(t.array(t.string))), ], - decode=lambda slots: { - "schemaVersion": slots[0], - "documentId": slots[1], - "authorRuntimeId": slots[2], - "title": slots[3], - "body": slots[4], - "tags": slots[5] if len(slots) > 5 and slots[5] is not None else [], - }, name="documentId", - payload_encoder=lambda value: msgpack.packb(value, use_bin_type=True), - payload_decoder=lambda payload: msgpack.unpackb(payload, raw=False), schema_id="cultcache.interop-note", schema_name="cultcache.interop-note", schema_version=INTEROP_SCHEMA_VERSION, content_hash="cultcache.interop-note", - canonical_schema_json='{"schemaName":"cultcache.interop-note","schemaVersion":"cultcache.interop_note.v1","members":[{"slot":0,"name":"SchemaVersion","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},{"slot":1,"name":"DocumentId","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":true},{"slot":2,"name":"AuthorRuntimeId","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},{"slot":3,"name":"Title","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},{"slot":4,"name":"Body","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},{"slot":5,"name":"Tags","type":"System.String[]","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}', compatible_schema_ids=("cultcache.interop-note",), - members=( - CultCacheSchemaCatalogMember(0, "SchemaVersion", "System.String"), - CultCacheSchemaCatalogMember(1, "DocumentId", "System.String", is_name=True), - CultCacheSchemaCatalogMember(2, "AuthorRuntimeId", "System.String"), - CultCacheSchemaCatalogMember(3, "Title", "System.String"), - CultCacheSchemaCatalogMember(4, "Body", "System.String"), - CultCacheSchemaCatalogMember(5, "Tags", "System.String[]"), - ), ) diff --git a/packages/cultcache-py/src/cultcache_py/leaves.py b/packages/cultcache-py/src/cultcache_py/leaves.py new file mode 100644 index 00000000..2d76078e --- /dev/null +++ b/packages/cultcache-py/src/cultcache_py/leaves.py @@ -0,0 +1,294 @@ +"""Typed slot leaves: each owns its slot's C# type name, its write check and its read check, nil included. + +A leaf is a `SlotCodec`. It turns a Python value into the tree the reference runtimes write and back, refusing a +kind the C# type would refuse. Refusals read ', found ' and never carry the value; the +record codec prefixes them with the type, slot key and field, a list with the element, a union with its arm. + +Several names here (`bytes`, `list`, `any`) shadow builtins inside this module on purpose: write +`from cultcache_py import leaves as t` and use `t.string`, `t.list(t.string)`. +""" + +from __future__ import annotations + +import math +import struct +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, Callable, NoReturn + +try: + from msgpack import ExtType as _ExtType +except ModuleNotFoundError: + _ExtType = () # type: ignore[assignment,misc] + +_bool, _bytes, _list, _int, _float, _str = bool, bytes, list, int, float, str +_BYTE_LIKE = (_bytes, bytearray, memoryview) +_INT32 = (-(2**31), 2**31 - 1) +_INT64 = (-(2**63), 2**63 - 1) +_WRITABLE_INT = (-(2**63), 2**64 - 1) + + +@dataclass(frozen=True) +class SlotCodec: + """The leaf of one slot: how a value is written and read, and the C# type name the catalog carries for it. + + `type_name` is the reference's name for the slot's type (None when only the consumer knows it: a record or union + without `type_name=`, which the catalog lists as System.Object). `value_type` is True for a C# value type, which + the reference makes `Nullable` when the slot may be nil.""" + + encode: Callable[[Any], Any] + decode: Callable[[Any], Any] + type_name: str | None = None + value_type: bool = False + + +class _Derived(SlotCodec): + """A leaf whose name comes from another leaf, read when the catalog is built, so a `lazy` target defined later resolves.""" + + def __init__(self, encode: Callable[[Any], Any], decode: Callable[[Any], Any], name: Callable[[], str | None], + value_type: Callable[[], _bool] = lambda: False) -> None: + object.__setattr__(self, "encode", encode) + object.__setattr__(self, "decode", decode) + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_value_type", value_type) + + @property # type: ignore[override] + def type_name(self) -> str | None: + return self._name() # type: ignore[attr-defined,no-any-return] + + @property # type: ignore[override] + def value_type(self) -> _bool: + return self._value_type() # type: ignore[attr-defined,no-any-return] + + +@dataclass(frozen=True) +class Float32: + """A float32 the float32 leaf writes, already rounded to float32 (an overflow is infinity, as the C# cast): the + payload packer writes it as MessagePack float 32.""" + + value: float + + +def _kind(raw: Any) -> str: + if raw is None: + return "nil" + if isinstance(raw, _bool): + return "boolean" + if isinstance(raw, _int): + return "integer" + if isinstance(raw, (_float, Float32)): + return "float" + if isinstance(raw, _str): + return "string" + if isinstance(raw, _BYTE_LIKE): + return "bytes" + if isinstance(raw, _ExtType): # a namedtuple, so before the arrays + return "ext" + if isinstance(raw, (_list, tuple)): + return "array" + return "map" if isinstance(raw, Mapping) else "other" + + +def _refuse(what: str, found: str) -> NoReturn: + raise ValueError(f"expected {what}, found {found}") + + +def _is_int(raw: Any) -> _bool: + return isinstance(raw, _int) and not isinstance(raw, _bool) + + +def _string_encode(value: Any) -> Any: + if not isinstance(value, _str): + _refuse("a string", _kind(value)) + return value + + +string = SlotCodec(_string_encode, _string_encode, "System.String") + + +def _boolean_check(value: Any) -> Any: + if not isinstance(value, _bool): + _refuse("a boolean", _kind(value)) + return value + + +boolean = SlotCodec(_boolean_check, _boolean_check, "System.Boolean", True) + + +def _bytes_encode(value: Any) -> Any: + if not isinstance(value, _BYTE_LIKE): + _refuse("bytes", _kind(value)) + return value if isinstance(value, _bytes) else _bytes(value) + + +def _bytes_decode(raw: Any) -> Any: + # The reference's byte[] reads a MessagePack str as its UTF-8 bytes and writes bin. + if isinstance(raw, _str): + return raw.encode("utf-8") + if not isinstance(raw, _bytes): + _refuse("bytes", _kind(raw)) + return raw + + +bytes = SlotCodec(_bytes_encode, _bytes_decode, "System.Byte[]") + + +def _integer(type_name: str, what: str, low: _int, high: _int) -> SlotCodec: + def check(value: Any) -> Any: + if not _is_int(value): + _refuse(what, _kind(value)) + if not low <= value <= high: + _refuse(what, "an integer out of range") + return value + + return SlotCodec(check, check, type_name, True) + + +int32 = _integer("System.Int32", "an int32", *_INT32) +int64 = _integer("System.Int64", "an int64", *_INT64) + + +def _round_to_float32(value: _float) -> _float: + try: + return struct.unpack(">f", struct.pack(">f", value))[0] # type: ignore[no-any-return] + except OverflowError: + return math.copysign(math.inf, value) + + +def _int_to_float32(value: _int) -> _float: + """An integer to the nearest float32 in one rounding, ties to even, as the reference's cast does: through a double + first, the two roundings can disagree past 2**53.""" + if -(2**53) <= value <= 2**53: + return _round_to_float32(_float(value)) + magnitude = -value if value < 0 else value + excess = magnitude.bit_length() - 24 + kept = magnitude >> excess + rest = magnitude - (kept << excess) + half = 1 << (excess - 1) + rounded = _float((kept + 1 if rest > half or (rest == half and kept & 1) else kept) << excess) + return -rounded if value < 0 else rounded + + +def _checked_int(what: str, value: _int) -> _int: + if not _WRITABLE_INT[0] <= value <= _WRITABLE_INT[1]: + _refuse(what, "an integer out of range") + return value + + +def _number(what: str, value: Any) -> _float: + """A number as a double: a float as it is, an integer (within the 64-bit range) rounded once.""" + if isinstance(value, _float): + return value + if not _is_int(value): + _refuse(what, _kind(value)) + return _float(_checked_int(what, value)) + + +def _float32_value(value: Any) -> _float: + """The float32 a number rounds to, in one rounding, as the reference's cast does.""" + if _is_int(value): + return _int_to_float32(_checked_int("a float32", value)) + return _round_to_float32(_number("a float32", value)) + + +def _float32_encode(value: Any) -> Any: + return Float32(_float32_value(value)) + + +def _float64_check(value: Any) -> Any: + return _number("a float64", value) + + +float32 = SlotCodec(_float32_encode, _float32_value, "System.Single", True) +float64 = SlotCodec(_float64_check, _float64_check, "System.Double", True) + + +def _refuse_ext(tree: Any) -> None: + """The reference reads an `object` slot as a plain value or a timestamp; any other extension is refused.""" + stack = [tree] + while stack: + node = stack.pop() + if isinstance(node, _ExtType): + _refuse("an object", "ext") + if isinstance(node, Mapping): + stack.extend(node.keys()) + stack.extend(node.values()) + elif isinstance(node, (_list, tuple)): + stack.extend(node) + + +def _any_check(value: Any) -> Any: + _refuse_ext(value) + return value + + +any = SlotCodec(_any_check, _any_check, "System.Object") + + +def _sequence(item: SlotCodec, wrap: Callable[[str], str]) -> SlotCodec: + def encode(value: Any) -> Any: + if not isinstance(value, (_list, tuple)): + _refuse("an array", _kind(value)) + out = [] + for index, element in enumerate(value): + try: + out.append(item.encode(element)) + except ValueError as error: + raise ValueError(f"element {index}: {error}") from None + return out + + def decode(raw: Any) -> Any: + if not isinstance(raw, _list): + _refuse("an array", _kind(raw)) + out = [] + for index, element in enumerate(raw): + try: + out.append(item.decode(element)) + except ValueError as error: + raise ValueError(f"element {index}: {error}") from None + return out + + return _Derived(encode, decode, lambda: None if item.type_name is None else wrap(item.type_name)) + + +def list(item: SlotCodec) -> SlotCodec: + """C# `List`.""" + return _sequence(item, lambda name: f"System.Collections.Generic.List<{name}>") + + +def array(item: SlotCodec) -> SlotCodec: + """C# `T[]`: the same wire as `list`.""" + return _sequence(item, lambda name: f"{name}[]") + + +def nullable(inner: SlotCodec) -> SlotCodec: + """A slot that may be nil: None both ways, otherwise `inner`. A C# reference type (string, byte[], List, array, + class, interface) is nullable as it is; a value type is `Nullable`.""" + + def name() -> str | None: + if inner.type_name is not None and inner.value_type: + return f"System.Nullable<{inner.type_name}>" + return inner.type_name + + return _Derived( + lambda value: None if value is None else inner.encode(value), + lambda raw: None if raw is None else inner.decode(raw), + name, + ) + + +def lazy(get: Callable[[], SlotCodec]) -> SlotCodec: + """A leaf that names its target on first use, so a union can refer to itself or to a definition made later.""" + resolved: _list[SlotCodec] = [] + + def target() -> SlotCodec: + if not resolved: + resolved.append(get()) + return resolved[0] + + return _Derived( + lambda value: target().encode(value), + lambda raw: target().decode(raw), + lambda: target().type_name, + lambda: target().value_type, + ) diff --git a/packages/cultcache-py/tests/test_typed_leaves.py b/packages/cultcache-py/tests/test_typed_leaves.py new file mode 100644 index 00000000..9b432efd --- /dev/null +++ b/packages/cultcache-py/tests/test_typed_leaves.py @@ -0,0 +1,733 @@ +from __future__ import annotations + +import math +import struct +import unittest +from pathlib import Path + +import msgpack + +from cultcache_py import ( + CultCache, + SingleFileMessagePackBackingStore, + UnionArm, + database_entry_field, + define_database_entry_type, + define_record_codec, + define_union, + leaves as t, +) +from cultcache_py.documents import _pack_tree +from cultcache_py.interop import interop_note_document + +VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "numeric-leaves" +CANARY = "CANARY-leaf-9d41" +REFERENCE = "GameCult.Caching.Tests.NumericLeafVectorTests+" + + +def _vector(name: str) -> list: + return msgpack.unpackb((VECTORS / name).read_bytes(), raw=False) + + +def _slot(leaf: t.SlotCodec): + return define_database_entry_type("tests.leaf", [database_entry_field("v", 0, codec=leaf)]) + + +def _doubles(bits: bytes) -> float: + return struct.unpack(">d", bits)[0] + + +def _is_signalling_float32(payload: bytes) -> bool: + if len(payload) != 6 or payload[1] != 0xCA: + return False + word = struct.unpack(">I", payload[2:])[0] + return word & 0x7F800000 == 0x7F800000 and word & 0x007FFFFF != 0 and word & 0x00400000 == 0 + + +def _kinded(node): + """A decoded tree with its kinds kept: bool, int and float never compare equal; numeric width is not part of it.""" + if isinstance(node, bool): + return ("bool", node) + if isinstance(node, int): + return ("int", node) + if isinstance(node, float): + return ("float", "nan" if math.isnan(node) else struct.pack(">d", node)) + if isinstance(node, (list, tuple)): + return ("array", [_kinded(item) for item in node]) + if isinstance(node, dict): + return ("map", [(_kinded(k), _kinded(v)) for k, v in node.items()]) + return (type(node).__name__, node) + + +NUMERIC = {"int32": t.int32, "int64": t.int64, "float32": t.float32, "float64": t.float64} + + +class NumericVectorTests(unittest.TestCase): + def test_reads_accept_refuse_and_round_exactly_as_the_reference(self) -> None: + rows = _vector("reads.msgpack") + self.assertEqual(len(rows), 160) + accepted = refused = 0 + for leaf, payload, was_accepted, bits in rows: + label = f"{leaf} {payload.hex()}" + document = _slot(NUMERIC[leaf]) + if not was_accepted: + with self.assertRaises(ValueError, msg=label): + document.decode_payload(payload) + refused += 1 + continue + read = document.decode_payload(payload)["v"] + if leaf.startswith("float"): + if math.isnan(_doubles(bits)): + self.assertTrue(math.isnan(read), label) + else: + self.assertEqual(struct.pack(">d", read), bits, label) + else: + self.assertEqual(read, int.from_bytes(bits, "big", signed=True), label) + accepted += 1 + self.assertEqual((accepted + refused, accepted > 40, refused > 40), (160, True, True)) + + def test_writes_are_the_bytes_the_reference_writes(self) -> None: + rows = _vector("writes.msgpack") + self.assertEqual(len(rows), 46) + for leaf, text, expected in rows: + value = _doubles(bytes.fromhex(text)) if leaf.startswith("float") else int(text) + self.assertEqual(_slot(NUMERIC[leaf]).encode_payload({"v": value}), expected, f"{leaf} {text}") + + def test_the_mixed_record_is_written_byte_for_byte_and_read_back(self) -> None: + mixed = define_database_entry_type( + "tests.mixed", + [ + database_entry_field("id", 0, codec=t.string), + database_entry_field("i", 1, codec=t.int32), + database_entry_field("l", 2, codec=t.int64), + database_entry_field("f", 3, codec=t.float32), + database_entry_field("d", 4, codec=t.float64), + ], + ) + committed = (VECTORS / "mixed-record.msgpack").read_bytes() + value = {"id": "n1", "i": 7, "l": 8, "f": 1.5, "d": 2.5} + self.assertEqual(mixed.encode_payload(value), committed) + self.assertEqual(mixed.decode_payload(committed), value) + + def test_float32_infinities_and_quiet_nans_keep_their_bits_and_signalling_ones_come_back_quieted(self) -> None: + rows = _vector("float32-nan.msgpack") + self.assertEqual(len(rows), 6) + document = _slot(t.float32) + quieted = 0 + for payload, reference in rows: + written = document.encode_payload(document.decode_payload(payload)) + if _is_signalling_float32(payload): + # follow_up py-float32-snan-quieted: msgpack's unpacker quiets the NaN before the leaf sees it. + self.assertEqual(reference, payload, "the reference keeps the signalling bits") + word = struct.unpack(">I", payload[2:])[0] | 0x00400000 + self.assertEqual(written, b"\x91\xca" + struct.pack(">I", word)) + quieted += 1 + else: + self.assertEqual(written, reference, payload.hex()) + self.assertEqual(quieted, 3) + + +def _inner_record() -> t.SlotCodec: + return define_record_codec( + "LeafInner", + [ + database_entry_field("n", 0, default="", codec=t.string), + database_entry_field("w", 1, default=0.0, codec=t.float32), + ], + type_name=REFERENCE + "LeafInner", + ) + + +def _shape() -> t.SlotCodec: + circle = define_record_codec( + "LeafCircle", [database_entry_field("r", 0, default=0.0, codec=t.float32)], type_name=REFERENCE + "LeafCircle" + ) + return define_union("LeafShape", {0: ("LeafCircle", circle)}, type_name=REFERENCE + "ILeafShape") + + +# The C# slot types of leaves-reads.msgpack, each as a reference-type slot that may be null (nullable) and bare. +def _reference_leaves(nullable: bool) -> dict[str, t.SlotCodec]: + inner, shape = _inner_record(), _shape() + wrap = t.nullable if nullable else (lambda leaf: leaf) + return { + "string": wrap(t.string), + "bytes": wrap(t.bytes), + "list-string": wrap(t.list(t.string)), + "array-string": wrap(t.array(t.string)), + "list-int32": wrap(t.list(t.int32)), + "record": wrap(inner), + "list-record": wrap(t.list(inner)), + "union": wrap(shape), + } + + +class LeavesReadsVectorTests(unittest.TestCase): + NIL = bytes.fromhex("91c0") + + def _check(self, leaf_name: str, leaf: t.SlotCodec, rows: list, *, nil_refused: bool = False) -> int: + document = _slot(leaf) + checked = 0 + for name, payload, accepted, rewrite in rows: + if name != leaf_name: + continue + label = f"{name} {payload.hex()}" + if not accepted or (nil_refused and payload == self.NIL): + with self.assertRaises(ValueError, msg=label): + document.decode_payload(payload) + else: + value = document.decode_payload(payload) + self.assertEqual(document.encode_payload(value), rewrite, label) + checked += 1 + return checked + + def test_reference_type_leaves_held_as_nullable_read_and_rewrite_exactly_as_csharp(self) -> None: + rows = _vector("leaves-reads.msgpack") + self.assertEqual(len(rows), 308) + extra = {"nullable-int32": t.nullable(t.int32)} + for name, leaf in {**_reference_leaves(True), **extra}.items(): + self.assertEqual(self._check(name, leaf, rows), 28, name) + self.assertTrue(any(row[0] == name and row[2] for row in rows) and any(row[0] == name and not row[2] for row in rows), name) + + def test_the_bare_leaf_accepts_the_same_rows_minus_nil(self) -> None: + rows = _vector("leaves-reads.msgpack") + for name, leaf in _reference_leaves(False).items(): + self.assertEqual(self._check(name, leaf, rows, nil_refused=True), 28, name) + + def test_boolean_is_a_bare_value_leaf(self) -> None: + self.assertEqual(self._check("boolean", t.boolean, _vector("leaves-reads.msgpack")), 28) + + def test_any_reads_what_the_reference_reads_and_rewrites_to_the_same_value(self) -> None: + # ruling any-rewrite-binds-kind-and-value: the kind and exact value of the rewrite, recursively; numeric width is free ("05" -> "cc05"). + rows = _vector("leaves-reads.msgpack") + document = _slot(t.any) + count = 0 + for name, payload, accepted, rewrite in rows: + if name != "any": + continue + label = f"any {payload.hex()}" + if not accepted: + with self.assertRaises(ValueError, msg=label): + document.decode_payload(payload) + else: + written = document.encode_payload(document.decode_payload(payload)) + self.assertEqual(_kinded(msgpack.unpackb(written, raw=False)), _kinded(msgpack.unpackb(rewrite, raw=False)), label) + count += 1 + self.assertEqual(count, 28) + + def test_the_any_comparison_tells_bool_int_and_float_apart_and_ignores_only_numeric_width(self) -> None: + # ruling any-rewrite-binds-kind-and-value: Python's == makes True == 1 == 1.0; _kinded does not. + self.assertNotEqual(_kinded([True]), _kinded([1])) + self.assertNotEqual(_kinded([1]), _kinded([1.0])) + self.assertNotEqual(_kinded({"k": [5]}), _kinded({"k": [5.0]})) + self.assertNotEqual(_kinded([0.0]), _kinded([-0.0])) + self.assertNotEqual(_kinded(["a"]), _kinded([b"a"])) + self.assertEqual(_kinded(msgpack.unpackb(bytes.fromhex("9105"))), _kinded(msgpack.unpackb(bytes.fromhex("91cc05")))) + self.assertEqual( + _kinded(msgpack.unpackb(bytes.fromhex("91ca3fc00000"))), _kinded(msgpack.unpackb(bytes.fromhex("91cb3ff8000000000000"))) + ) + + def test_any_refuses_an_extension_that_is_not_a_timestamp_wherever_it_nests(self) -> None: + document = _slot(t.any) + for payload in (bytes.fromhex("91d40100"), bytes.fromhex("9191d40100"), bytes.fromhex("9181a161d40100")): + with self.assertRaises(ValueError, msg=payload.hex()): + document.decode_payload(payload) + with self.assertRaises(ValueError): + t.any.encode({"k": [msgpack.ExtType(1, b"\x00")]}) + with self.assertRaises(ValueError) as caught: + t.any.decode(msgpack.ExtType(1, b"\x00")) + self.assertEqual(str(caught.exception), "expected an object, found ext") + self.assertIsNone(t.any.decode(None)) + self.assertIsNone(t.any.encode(None)) + + +def _leaf_record_document(): + inner, shape = _inner_record(), _shape() + document = define_database_entry_type( + "cultcache.leaf-record", + [ + database_entry_field("id", 0, codec=t.string), + database_entry_field("flag", 1, codec=t.boolean), + database_entry_field("blob", 2, codec=t.bytes), + database_entry_field("i", 3, codec=t.int32), + database_entry_field("l", 4, codec=t.int64), + database_entry_field("f", 5, codec=t.float32), + database_entry_field("d", 6, codec=t.float64), + database_entry_field("maybe_i", 7, codec=t.nullable(t.int32)), + database_entry_field("tags", 8, codec=t.list(t.string)), + database_entry_field("names", 9, codec=t.array(t.string)), + database_entry_field("any", 10, codec=t.any), + database_entry_field("nested", 11, codec=t.nullable(inner)), + database_entry_field("items", 12, codec=t.list(inner)), + database_entry_field("shape", 13, codec=t.nullable(shape)), + database_entry_field("floats", 14, codec=t.array(t.float32)), + database_entry_field("maybe_f", 15, codec=t.nullable(t.float32)), + ], + name="id", + ) + return document + + +class LeafRecordCatalogTests(unittest.TestCase): + def test_a_python_mirror_derives_the_catalog_the_csharp_store_writes_and_the_payload_round_trips(self) -> None: + document = _leaf_record_document() + members, payload = _vector("leaf-record.msgpack") + derived = [ + [m.slot, m.member_name, m.type_name, m.is_reference, m.is_many, m.target_schema_name, m.is_name, m.index_alias] + for m in document.catalog_entry().members + ] + self.assertEqual(derived, members) + self.assertEqual(document.encode_payload(document.decode_payload(payload)), payload) + + def test_nullable_names_a_value_type_nullable_and_a_reference_type_as_itself(self) -> None: + self.assertEqual(t.nullable(t.int32).type_name, "System.Nullable") + self.assertEqual(t.nullable(t.boolean).type_name, "System.Nullable") + self.assertEqual(t.nullable(t.string).type_name, "System.String") + self.assertEqual(t.nullable(t.list(t.string)).type_name, "System.Collections.Generic.List") + self.assertEqual(t.array(t.nullable(t.int64)).type_name, "System.Nullable[]") + self.assertEqual(t.array(t.float32).type_name, "System.Single[]") + self.assertEqual(t.bytes.type_name, "System.Byte[]") + self.assertFalse(t.nullable(t.int32).value_type) + self.assertEqual([c.value_type for c in (t.string, t.boolean, t.bytes, t.int32, t.int64, t.float32, t.float64, t.any)], + [False, True, False, True, True, True, True, False]) + + def test_an_unnamed_record_or_union_is_listed_as_an_untyped_slot_and_a_list_of_one_too(self) -> None: + record = define_record_codec("Plain", [("a", 0)]) + self.assertIsNone(record.type_name) + self.assertIsNone(t.list(record).type_name) + self.assertIsNone(t.nullable(record).type_name) + document = define_database_entry_type( + "tests.unnamed", [database_entry_field("r", 0, codec=record), database_entry_field("l", 1, codec=t.list(record))] + ) + self.assertEqual([m.type_name for m in document.catalog_entry().members], ["System.Object", "System.Object"]) + + def test_a_recursive_union_through_lazy_encodes_decodes_and_names_its_type(self) -> None: + link = define_record_codec( + "Link", [database_entry_field("next", 0, codec=t.nullable(t.lazy(lambda: chain)))], type_name="Tests.Link" + ) + chain = define_union("Chain", {0: ("Link", link), 1: ("End", None)}, type_name="Tests.Chain") + self.assertEqual(t.lazy(lambda: chain).type_name, "Tests.Chain") + self.assertEqual(t.nullable(t.lazy(lambda: chain)).type_name, "Tests.Chain") + value = UnionArm("Link", {"next": UnionArm("Link", {"next": UnionArm("End")})}) + written = chain.encode(value) + self.assertEqual(written, [0, [[0, [[1, []]]]]]) + self.assertEqual(chain.decode(written), value) + self.assertIs(t.lazy(lambda: t.int32).value_type, True) + document = define_database_entry_type("tests.chain", [database_entry_field("root", 0, codec=t.nullable(chain))]) + self.assertEqual(document.catalog_entry().members[0].type_name, "Tests.Chain") + + def test_a_lazy_leaf_resolves_its_target_once(self) -> None: + calls = [] + leaf = t.lazy(lambda: calls.append(1) or t.int32) + self.assertEqual(calls, []) + leaf.encode(1) + leaf.decode(2) + leaf.type_name + self.assertEqual(calls, [1]) + + +class SlotNilAndDefaultTests(unittest.TestCase): + def test_a_present_nil_goes_to_the_leaf_and_a_default_fills_only_a_slot_past_the_payloads_end(self) -> None: + document = define_database_entry_type( + "tests.defaults", + [ + database_entry_field("count", 0, default=0, codec=t.int32), + database_entry_field("label", 1, default="", codec=t.string), + database_entry_field("maybe", 2, default="dflt", codec=t.nullable(t.string)), + database_entry_field("tags", 3, default=list, codec=t.list(t.string)), + ], + ) + self.assertEqual( + document.decode_payload(msgpack.packb([3, "x", None, ["a"]])), {"count": 3, "label": "x", "maybe": None, "tags": ["a"]} + ) + self.assertEqual(document.decode_payload(msgpack.packb([3])), {"count": 3, "label": "", "maybe": "dflt", "tags": []}) + self.assertIsNot( + document.decode_payload(msgpack.packb([3]))["tags"], document.decode_payload(msgpack.packb([3]))["tags"] + ) + for bad in ([None], [3, None], [3, "x", None, None]): + with self.assertRaises(ValueError, msg=str(bad)): + document.decode_payload(msgpack.packb(bad)) + + def test_a_write_missing_a_non_nullable_field_is_refused_naming_the_slot(self) -> None: + document = define_database_entry_type( + "tests.required", + [database_entry_field("count", 0, codec=t.int32), database_entry_field("label", 1, codec=t.nullable(t.string))], + ) + self.assertEqual(document.encode_payload({"count": 1}), msgpack.packb([1, None])) + with self.assertRaises(ValueError) as caught: + document.encode_payload({"label": "x"}) + self.assertEqual(str(caught.exception), "tests.required slot 0 (count): expected an int32, found nil") + with self.assertRaises(ValueError) as nil_record: + define_record_codec("R", [("a", 0)]).encode(None) + self.assertIn("found nil", str(nil_record.exception)) + + def test_an_untyped_definition_keeps_its_catalog_and_bytes_and_reads_a_present_nil_as_none(self) -> None: + document = define_database_entry_type("tests.untyped", [("name", 0), ("owner_id", 1, "none"), ("tags", 3, list)]) + self.assertEqual([(m.slot, m.member_name, m.type_name) for m in document.catalog_entry().members], + [(0, "Name", "System.Object"), (1, "OwnerId", "System.Object"), (3, "Tags", "System.Object")]) + # Values computed with the code at base 4ec4bfb7. + self.assertEqual(document.encode_payload({"name": "n", "owner_id": "o", "tags": ["a"]}).hex(), "94a16ea16fc091a161") + self.assertEqual(document.decode_payload(bytes.fromhex("94a16ea16fc091a161")), {"name": "n", "owner_id": "o", "tags": ["a"]}) + self.assertEqual(document.decode_payload(bytes.fromhex("94a16ec0c0c0")), {"name": "n", "owner_id": None, "tags": None}) + self.assertEqual(document.decode_payload(bytes.fromhex("91a16e")), {"name": "n", "owner_id": "none", "tags": []}) + self.assertEqual(_pack_tree(["x", 1.5, {"k": [b"y"]}]), msgpack.packb(["x", 1.5, {"k": [b"y"]}], use_bin_type=True)) + + def test_the_interop_note_catalog_is_the_one_the_hand_written_definition_had(self) -> None: + entry = interop_note_document.catalog_entry() + names = [(m.slot, m.member_name, m.type_name, m.is_name) for m in entry.members] + self.assertEqual( + names, + [(0, "SchemaVersion", "System.String", False), (1, "DocumentId", "System.String", True), + (2, "AuthorRuntimeId", "System.String", False), (3, "Title", "System.String", False), + (4, "Body", "System.String", False), (5, "Tags", "System.String[]", False)], + ) + # Captured from the hand-written definition at base 4ec4bfb7. + self.assertEqual(entry.canonical_schema_json, _BASE_INTEROP_CATALOG) + self.assertEqual( + (entry.schema_id, entry.schema_name, entry.schema_version, entry.content_hash, entry.compatible_schema_ids), + ("cultcache.interop-note", "cultcache.interop-note", "cultcache.interop_note.v1", "cultcache.interop-note", + ("cultcache.interop-note",)), + ) + note = {"schemaVersion": "cultcache.interop_note.v1", "documentId": "note:py", "authorRuntimeId": "py", + "title": "t", "body": "b", "tags": ["py", "interop"]} + self.assertEqual(interop_note_document.encode_payload(note).hex(), _BASE_INTEROP_PAYLOAD) + self.assertEqual(interop_note_document.decode_payload(bytes.fromhex(_BASE_INTEROP_PAYLOAD)), note) + self.assertEqual(interop_note_document.decode_payload(msgpack.packb(["v", "d", "a", "t", "b"]))["tags"], []) + + +_BASE_INTEROP_CATALOG = ( + '{"schemaName":"cultcache.interop-note","schemaVersion":"cultcache.interop_note.v1","members":[' + '{"slot":0,"name":"SchemaVersion","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},' + '{"slot":1,"name":"DocumentId","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":true},' + '{"slot":2,"name":"AuthorRuntimeId","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},' + '{"slot":3,"name":"Title","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},' + '{"slot":4,"name":"Body","type":"System.String","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false},' + '{"slot":5,"name":"Tags","type":"System.String[]","isReference":false,"many":false,"targetSchemaName":null,"indexAlias":null,"isName":false}]}' +) +_BASE_INTEROP_PAYLOAD = "96" + "".join( + msgpack.packb(v, use_bin_type=True).hex() + for v in ("cultcache.interop_note.v1", "note:py", "py", "t", "b", ["py", "interop"]) +) + + +class LeafRuleTests(unittest.TestCase): + def _refusal(self, run) -> str: + with self.assertRaises(ValueError) as caught: + run() + return str(caught.exception) + + def test_refusals_name_the_slot_and_the_kind_found_and_never_the_value(self) -> None: + cases = [ + (t.int32, CANARY, "expected an int32, found string"), + (t.int64, 1.5, "expected an int64, found float"), + (t.string, b"bin", "expected a string, found bytes"), + (t.boolean, 1, "expected a boolean, found integer"), + (t.bytes, 5, "expected bytes, found integer"), + (t.float32, CANARY, "expected a float32, found string"), + (t.float64, [CANARY], "expected a float64, found array"), + (t.list(t.string), {CANARY: 1}, "expected an array, found map"), + (t.string, None, "expected a string, found nil"), + (t.int32, True, "expected an int32, found boolean"), + (t.int32, 2**31, "expected an int32, found an integer out of range"), + ] + for leaf, value, expected in cases: + document = _slot(leaf) + with self.subTest(expected=expected, direction="write"): + self.assertEqual(self._refusal(lambda: document.encode_payload({"v": value})), f"tests.leaf slot 0 (v): {expected}") + with self.subTest(expected=expected, direction="read"): + payload = msgpack.packb([value], use_bin_type=True) + message = self._refusal(lambda: document.decode_payload(payload)) + self.assertTrue(message.startswith("tests.leaf slot 0 (v): expected "), message) + self.assertNotIn(CANARY, message) + + def test_a_list_element_refusal_names_the_element_and_a_union_arm_refusal_names_the_arm(self) -> None: + self.assertEqual( + self._refusal(lambda: t.list(t.int32).decode([1, "x"])), "element 1: expected an int32, found string" + ) + self.assertEqual( + self._refusal(lambda: t.array(t.list(t.int32)).encode([[1], [2, None]])), + "element 1: element 1: expected an int32, found nil", + ) + arm = define_union("U", {3: ("A", define_record_codec("A", [database_entry_field("n", 0, codec=t.int32)]))}) + self.assertEqual( + self._refusal(lambda: arm.decode([3, [CANARY]])), "Union 'U' arm 3: A slot 0 (n): expected an int32, found string" + ) + self.assertEqual( + self._refusal(lambda: arm.encode(UnionArm("A", {"n": None}))), "Union 'U' arm 3: A slot 0 (n): expected an int32, found nil" + ) + + def test_string_boolean_and_bytes_domains(self) -> None: + self.assertEqual(t.string.encode("é"), "é") + self.assertIs(t.boolean.decode(True), True) + self.assertEqual(t.bytes.encode(bytearray(b"ab")), b"ab") + self.assertEqual(t.bytes.encode(memoryview(b"ab")), b"ab") + self.assertIs(type(t.bytes.encode(bytearray(b"ab"))), bytes) + self.assertEqual(t.bytes.decode("é"), "é".encode("utf-8")) + self.assertEqual(t.bytes.decode(b"\x00"), b"\x00") + self.assertEqual(t.bytes.decode(""), b"") + for refuse in (lambda: t.bytes.decode(5), lambda: t.bytes.encode("str")): + self.assertIn("expected bytes", self._refusal(refuse)) + + def test_integer_leaves_hold_their_ranges_and_refuse_floats_and_booleans(self) -> None: + self.assertEqual((t.int32.encode(-(2**31)), t.int32.encode(2**31 - 1)), (-(2**31), 2**31 - 1)) + self.assertEqual((t.int64.decode(-(2**63)), t.int64.decode(2**63 - 1)), (-(2**63), 2**63 - 1)) + for leaf, value in ((t.int32, -(2**31) - 1), (t.int32, 2**31), (t.int64, -(2**63) - 1), (t.int64, 2**63), (t.int64, 2**64 - 1)): + self.assertIn("out of range", self._refusal(lambda: leaf.decode(value))) + self.assertIn("out of range", self._refusal(lambda: leaf.encode(value))) + for bad in (1.0, True, False, "1", None): + self.assertIn("expected an int32", self._refusal(lambda: t.int32.decode(bad))) + + def test_float_leaves_take_ints_and_floats_round_once_and_refuse_booleans(self) -> None: + self.assertEqual(t.float32.decode(0.1), struct.unpack(">f", struct.pack(">f", 0.1))[0]) + self.assertEqual(t.float32.decode(2**24 + 1), float(2**24)) + self.assertEqual(t.float32.decode(1e300), math.inf) + self.assertEqual(t.float32.decode(-1e300), -math.inf) + self.assertEqual(t.float64.decode(2**53 + 1), float(2**53)) + # past 2**53 a double then a float32 rounding can differ from one rounding: 2**62 + 2**38 + 1 is above the tie. + value = 2**62 + 2**38 + 1 + self.assertEqual(t.float32.decode(value), float(2**62 + 2**39)) + self.assertEqual(t.float32.decode(-value), -float(2**62 + 2**39)) + tie_even, tie_odd = 2**62 + 2**38, 2**62 + 2**39 + 2**38 + self.assertEqual(t.float32.decode(tie_even), float(2**62)) + self.assertEqual(t.float32.decode(tie_odd), float(2**62 + 2**40)) + self.assertEqual(t.float32.encode(2**64 - 1), t.Float32(float(2**64))) + self.assertEqual(t.float32.encode(1.5), t.Float32(1.5)) + self.assertEqual(t.float64.encode(3), 3.0) + self.assertIs(type(t.float64.encode(3)), float) + for leaf in (t.float32, t.float64): + for bad in (True, "1", None, [1.0]): + self.assertIn("expected a float", self._refusal(lambda: leaf.decode(bad))) + self.assertIn("expected a float", self._refusal(lambda: leaf.encode(bad))) + self.assertIn("out of range", self._refusal(lambda: leaf.encode(2**64))) + self.assertIn("out of range", self._refusal(lambda: leaf.encode(-(2**63) - 1))) + + def test_list_and_array_write_lists_and_tuples_and_read_only_arrays(self) -> None: + self.assertEqual(t.list(t.int32).encode((1, 2)), [1, 2]) + self.assertEqual(t.array(t.string).decode(["a"]), ["a"]) + self.assertEqual(t.list(t.int32).encode([]), []) + self.assertIn("expected an array", self._refusal(lambda: t.list(t.int32).encode("12"))) + self.assertIn("expected an array", self._refusal(lambda: t.list(t.int32).decode((1,)))) + self.assertIn("expected an array", self._refusal(lambda: t.array(t.int32).decode(None))) + + def test_nullable_passes_none_and_otherwise_defers(self) -> None: + maybe = t.nullable(t.int32) + self.assertIsNone(maybe.encode(None)) + self.assertIsNone(maybe.decode(None)) + self.assertEqual((maybe.encode(0), maybe.decode(0)), (0, 0)) + self.assertIn("expected an int32", self._refusal(lambda: maybe.decode("x"))) + self.assertEqual((t.nullable(t.boolean).encode(False), t.nullable(t.string).decode("")), (False, "")) + + def test_a_field_codec_must_be_a_slot_codec(self) -> None: + self.assertIn("must be a SlotCodec", self._refusal(lambda: database_entry_field("a", 0, codec=lambda: t.int32))) # type: ignore[arg-type] + + +class LeafBoundaryTests(unittest.TestCase): + def _refusal(self, run) -> str: + with self.assertRaises(ValueError) as caught: + run() + return str(caught.exception) + + def test_float32_writes_a_subnormal_as_a_float_32_with_its_exact_bits(self) -> None: + document = _slot(t.float32) + for value, expected in ( + (1e-40, "91ca000116c2"), + (-1e-40, "91ca800116c2"), + (1.401298464324817e-45, "91ca00000001"), + (7e-46, "91ca00000000"), + (1.1754942106924411e-38, "91ca007fffff"), + ): + self.assertEqual(document.encode_payload({"v": value}).hex(), expected, repr(value)) + + def test_float_leaves_write_an_int_across_the_whole_64_bit_range_and_refuse_beyond_it(self) -> None: + for leaf in (t.float32, t.float64): + for ok in (3, 0, -(2**63), 2**64 - 1): + leaf.encode(ok) + for bad in (-(2**63) - 1, 2**64): + what = "a float32" if leaf is t.float32 else "a float64" + expected = f"expected {what}, found an integer out of range" + self.assertEqual(self._refusal(lambda: leaf.encode(bad)), expected) + self.assertEqual(self._refusal(lambda: leaf.decode(bad)), expected) + self.assertEqual(t.float32.encode(3), t.Float32(3.0)) + self.assertEqual(t.float32.encode(-(2**63)), t.Float32(-float(2**63))) + self.assertEqual(t.float64.encode(-(2**63)), -float(2**63)) + self.assertEqual(t.float64.encode(2**64 - 1), float(2**64)) + + def test_an_integer_past_2_to_53_rounds_to_float32_once_not_through_a_double(self) -> None: + # 2**53 + 2**29 + 1 is above the float32 tie, but rounding to a double first lands on the tie and rounds down. + value = 2**53 + 2**29 + 1 + self.assertEqual(t.float32.decode(value), float(2**53 + 2**30)) + self.assertEqual(t.float32.decode(-value), -float(2**53 + 2**30)) + self.assertEqual(t.float32.encode(value), t.Float32(float(2**53 + 2**30))) + self.assertEqual(t.float32.decode(2**53), float(2**53)) + self.assertEqual(t.float32.decode(-(2**53)), -float(2**53)) + + def test_the_kind_a_refusal_reports(self) -> None: + for value, kind in ((msgpack.ExtType(1, b""), "ext"), (object(), "other"), (t.Float32(1.0), "float"), + (bytearray(1), "bytes"), ((1,), "array"), ({}, "map"), (True, "boolean")): + self.assertEqual(self._refusal(lambda: t.int32.encode(value)), f"expected an int32, found {kind}") + + def test_definition_time_refusals_say_what_is_wrong(self) -> None: + self.assertEqual( + self._refusal(lambda: database_entry_field("a", 0, codec=object)), # type: ignore[arg-type] + "DatabaseEntry field codec must be a SlotCodec: a leaf from cultcache_py.leaves, or leaves.lazy", + ) + self.assertEqual( + self._refusal(lambda: define_union("Loose", {0: ("A", object)})), # type: ignore[dict-item] + "Union 'Loose' arm codecs must be SlotCodecs or None", + ) + self.assertEqual( + self._refusal(lambda: define_record_codec("R", [("a", 0)]).encode(None)), + "DatabaseEntry document type 'R': expected a record, found nil", + ) + + def test_value_objects_are_immutable_and_unnamed_by_default(self) -> None: + import dataclasses + + plain = t.SlotCodec(lambda v: v, lambda v: v) + self.assertIsNone(plain.type_name) + self.assertIs(plain.value_type, False) + self.assertIs(t.nullable(plain).value_type, False) + self.assertIs(t.list(plain).value_type, False) + self.assertIsNone(t.nullable(plain).type_name) + for frozen, field_name in ((plain, "encode"), (t.Float32(1.0), "value"), (t.nullable(t.int32), "encode")): + with self.assertRaises(dataclasses.FrozenInstanceError): + setattr(frozen, field_name, None) + + def test_a_definition_made_without_members_has_an_empty_catalog(self) -> None: + from cultcache_py.documents import DocumentDefinition + + self.assertEqual(DocumentDefinition(type="tests.bare").catalog_entry().members, ()) + + +class BytesAtTheDepthSeamTests(unittest.TestCase): + def test_bytes_bytearray_and_memoryview_leaves_at_depth_256_are_accepted_on_write_and_read(self) -> None: + blob = define_record_codec("Blob", [database_entry_field("data", 0, codec=t.list(t.bytes))]) + wrap = define_record_codec("Wrap", [database_entry_field("inner", 0, codec=t.nullable(t.lazy(lambda: deep_union)))]) + deep_union = define_union("Deep", {0: ("Wrap", wrap), 1: ("Blob", blob)}) + document = define_database_entry_type("tests.deep_bytes", [database_entry_field("root", 0, codec=deep_union)]) + for data in (b"\x01\x02", bytearray(b"\x01\x02"), memoryview(b"\x01\x02")): + value = UnionArm("Blob", {"data": [data]}) + for _ in range(126): + value = UnionArm("Wrap", {"inner": value}) + payload = document.encode_payload({"root": value}) + decoded = document.decode_payload(payload) + self.assertEqual(document.encode_payload(decoded), payload) + leaf = decoded["root"] + while leaf.name == "Wrap": + leaf = leaf.value["inner"] + self.assertEqual(leaf.value["data"], [b"\x01\x02"]) + + +class PackTreeTests(unittest.TestCase): + def test_a_float32_anywhere_in_the_tree_is_written_as_float_32_and_everything_else_as_packb_does(self) -> None: + tree = [t.Float32(1.5), {"k": t.Float32(0.1), "b": b"\x01", "s": "x", "n": None}, (t.Float32(2.0), [True, 7, 2.5])] + packer = msgpack.Packer(use_bin_type=True) + single = msgpack.Packer(use_bin_type=True, use_single_float=True) + expected = ( + packer.pack_array_header(3) + single.pack(1.5) + + packer.pack_map_header(4) + packer.pack("k") + single.pack(0.1) + packer.pack("b") + packer.pack(b"\x01") + + packer.pack("s") + packer.pack("x") + packer.pack("n") + packer.pack(None) + + packer.pack_array_header(2) + single.pack(2.0) + packer.pack([True, 7, 2.5]) + ) + self.assertEqual(_pack_tree(tree), expected) + self.assertEqual(_pack_tree(t.Float32(1.5)), bytes.fromhex("ca3fc00000")) + + def test_a_float32_overflow_is_infinity_and_a_value_nothing_can_pack_is_still_a_type_error(self) -> None: + self.assertEqual(_pack_tree([t.float32.encode(1e300)]), bytes.fromhex("91ca7f800000")) + self.assertEqual(_pack_tree([t.float32.encode(-1e300)]), bytes.fromhex("91caff800000")) + with self.assertRaises(TypeError): + _pack_tree([t.Float32(1.0), object()]) + with self.assertRaises(TypeError): + _pack_tree([object()]) + + def test_a_tree_with_no_float32_is_packb_unchanged(self) -> None: + tree = {"a": [1, 2.5, "x", b"y", None, True, {"b": -1}], "c": (3, 4)} + self.assertEqual(_pack_tree(tree), msgpack.packb(tree, use_bin_type=True)) + + def test_the_union_golden_scalar_direction_is_refused_as_the_reference_refuses_it(self) -> None: + from test_union_parity import node # the fixture with the C# golden's leaves + + with self.assertRaises(ValueError) as caught: + node.decode([1, ["hammer", 12]]) + self.assertEqual(str(caught.exception), "Union 'Node' arm 1: Text slot 1 (content): expected a string, found integer") + self.assertNotIn("hammer", str(caught.exception)) + self.assertEqual(node.decode([1, ["hammer", "x"]]), UnionArm("Text", {"id": "hammer", "content": "x"})) + + +PY_STORES = VECTORS / "py-store" +# slot -> the C# type name a wrong catalog gives it; the reference refuses each ("slot N changed type"). +WRONG_NAMES = {3: "System.Int64", 5: "System.Double", 7: "System.Int32", 14: "System.Double[]"} + + +def _python_authored_leaf_record() -> dict: + return { + "id": "py", "flag": False, "blob": bytearray([1, 255]), "i": -(2**31), "l": 2**63 - 1, "f": 0.1, "d": 1e-300, + "maybe_i": None, "tags": [], "names": ("\u00e9", "z"), "any": {"k": [1, True, None, 2.5, "s"], "n": -3}, + "nested": None, "items": [{"n": "one", "w": 2**53 + 2**29 + 1}, {"n": "two", "w": 1e-40}], + "shape": UnionArm("LeafCircle", {"r": 0.1}), "floats": [0.1, 3, -0.0, 1e39], "maybe_f": None, + } + + +def _write_store(path: Path, document, value: dict) -> None: + path.unlink(missing_ok=True) + cache = CultCache.builder().register_document_type(document).add_generic_store(SingleFileMessagePackBackingStore(path)).build() + cache.pull_all_backing_stores() + cache.put(document, value["id"], value) + + +def _wrong_catalog(document, slot: int): + import dataclasses + + good = tuple(document.catalog_entry().members) + return dataclasses.replace( + document, members=lambda: tuple(dataclasses.replace(m, type_name=WRONG_NAMES[slot]) if m.slot == slot else m for m in good) + ) + + +class PythonWrittenStoreVectorTests(unittest.TestCase): + """The stores under tests/vectors/numeric-leaves/py-store are written only by this class, with CULTCACHE_WRITE_VECTORS=1; + GameCult.Caching.Tests (PythonWrittenStoreTests) opens them with the C# reference and asserts every field.""" + + def test_the_committed_stores_are_what_the_definition_writes_and_read_back(self) -> None: + import os + import tempfile + + document = _leaf_record_document() + value = _python_authored_leaf_record() + if os.environ.get("CULTCACHE_WRITE_VECTORS"): + PY_STORES.mkdir(exist_ok=True) + _write_store(PY_STORES / "leaf-record.cc", document, value) + for slot in WRONG_NAMES: + _write_store(PY_STORES / f"wrong-slot-{slot}.cc", _wrong_catalog(document, slot), value) + with tempfile.TemporaryDirectory() as tmp: + fresh = Path(tmp) / "fresh.cc" + _write_store(fresh, document, value) + [committed] = SingleFileMessagePackBackingStore(PY_STORES / "leaf-record.cc").pull_all() + [again] = SingleFileMessagePackBackingStore(fresh).pull_all() + self.assertEqual(committed.payload, again.payload) + self.assertEqual(committed.catalog_entry.members, again.catalog_entry.members) + self.assertEqual(committed.payload, document.encode_payload(value)) + for slot, name in WRONG_NAMES.items(): + [wrong] = SingleFileMessagePackBackingStore(PY_STORES / f"wrong-slot-{slot}.cc").pull_all() + self.assertEqual(wrong.catalog_entry.members[slot].type_name, name) + self.assertEqual(wrong.payload, committed.payload) + + +class StoreRoundTripTests(unittest.TestCase): + def test_a_float32_slot_survives_a_single_file_store(self) -> None: + import tempfile + + document = define_database_entry_type( + "tests.store", [database_entry_field("id", 0, codec=t.string), database_entry_field("f", 1, codec=t.float32)], name="id" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.cc" + cache = CultCache.builder().register_document_type(document).add_generic_store(SingleFileMessagePackBackingStore(path)).build() + cache.pull_all_backing_stores() + cache.put(document, "a", {"id": "a", "f": 0.1}) + [envelope] = SingleFileMessagePackBackingStore(path).pull_all() + self.assertEqual(envelope.payload, bytes.fromhex("92a161ca3dcccccd")) + self.assertEqual(envelope.catalog_entry.members[1].type_name, "System.Single") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index 45d88458..e6fe6c2f 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from __future__ import annotations import collections import dataclasses @@ -17,6 +17,7 @@ from cultcache_py import ( CultCache, CultCachePayloadDepthError, DatabaseEntryField, + SlotCodec, SingleFileMessagePackBackingStore, UnionArm, database_entry_field, @@ -24,55 +25,92 @@ from cultcache_py import ( define_document_type, define_record_codec, define_union, + leaves as t, ) VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "union-parity" +_GOLDEN = "GameCult.Caching.Tests.UnionParityVectorTests+" +_node = t.nullable(t.lazy(lambda: node)) composition = define_record_codec( - "Composition", [("name", 0), database_entry_field("body", 1, codec=lambda: node)] + "Composition", + [database_entry_field("name", 0, codec=t.string), database_entry_field("body", 1, codec=_node)], + type_name=_GOLDEN + "Composition", +) +box = define_record_codec( + "Box", + [ + database_entry_field("id", 0, codec=t.string), + database_entry_field("children", 1, codec=t.list(t.lazy(lambda: node))), + ], + type_name=_GOLDEN + "Box", +) +text = define_record_codec( + "Text", + [database_entry_field("id", 0, codec=t.string), database_entry_field("content", 1, codec=t.string)], + type_name=_GOLDEN + "Text", ) -box = define_record_codec("Box", [("id", 0), database_entry_field("children", 1, codec=lambda: node, many=True)]) -text = define_record_codec("Text", [("id", 0), ("content", 1)]) use = define_record_codec( "Use", [ - ("id", 0), + database_entry_field("id", 0, codec=t.string), database_entry_field("composition", 1, codec=composition), - database_entry_field("slots", 2, codec=lambda: node, many=True), + database_entry_field("slots", 2, codec=t.list(t.nullable(t.lazy(lambda: node)))), ], + type_name=_GOLDEN + "Use", ) -image = define_record_codec("Image", [("id", 0), ("uri", 1), ("alt", 3)]) -wrap = define_record_codec("Wrap", [database_entry_field("inner", 0, codec=lambda: node)]) +image = define_record_codec( + "Image", + [ + database_entry_field("id", 0, codec=t.string), + database_entry_field("uri", 1, codec=t.string), + database_entry_field("alt", 3, codec=t.string), + ], + type_name=_GOLDEN + "Image", +) +wrap = define_record_codec("Wrap", [database_entry_field("inner", 0, codec=_node)], type_name=_GOLDEN + "Wrap") node = define_union( - "Node", {0: ("Box", box), 1: ("Text", text), 2: ("Use", use), 5: ("Image", image), 6: ("Wrap", wrap)} + "Node", + {0: ("Box", box), 1: ("Text", text), 2: ("Use", use), 5: ("Image", image), 6: ("Wrap", wrap)}, + type_name=_GOLDEN + "Node", ) outcome = define_union( "Outcome", { - 0: ("Pending", define_record_codec("Pending", [("message", 0)])), - 1: ("Completed", define_record_codec("Completed", [("source_version", 0)])), + 0: ("Pending", define_record_codec("Pending", [database_entry_field("message", 0, codec=t.string)])), + 1: ("Completed", define_record_codec("Completed", [database_entry_field("source_version", 0, codec=t.int32)])), 2: ("Denied", None), -2147483648: ("Floor", None), 2147483647: ("Ceiling", None), }, + type_name=_GOLDEN + "Outcome", ) -tool = define_record_codec("Tool", [("name", 0), ("uses", 1)]) -part = define_record_codec("Part", [("name", 0), ("count", 1)]) -gem = define_record_codec("Gem", [("name", 0)]) -item = define_union("Item", {0: ("Gem", gem), 1: ("Tool", tool), 2: ("Part", part)}) -crafted = define_union("CraftedItem", {0: ("Tool", tool), 1: ("Part", part)}) +tool = define_record_codec( + "Tool", + [database_entry_field("name", 0, codec=t.string), database_entry_field("uses", 1, codec=t.int32)], + type_name=_GOLDEN + "Tool", +) +part = define_record_codec( + "Part", + [database_entry_field("name", 0, codec=t.string), database_entry_field("count", 1, codec=t.int32)], + type_name=_GOLDEN + "Part", +) +gem = define_record_codec("Gem", [database_entry_field("name", 0, codec=t.string)], type_name=_GOLDEN + "Gem") +item = define_union("Item", {0: ("Gem", gem), 1: ("Tool", tool), 2: ("Part", part)}, type_name=_GOLDEN + "Item") +crafted = define_union("CraftedItem", {0: ("Tool", tool), 1: ("Part", part)}, type_name=_GOLDEN + "CraftedItem") surface_document = define_database_entry_type( "tests.union_surface", [ - ("id", 0), - ("version", 1), - database_entry_field("root", 2, codec=node), - database_entry_field("outcomes", 3, codec=outcome, many=True), - database_entry_field("part", 4, codec=item), - database_entry_field("crafted", 5, codec=crafted), - database_entry_field("missing", 6, codec=node), + database_entry_field("id", 0, codec=t.string), + database_entry_field("version", 1, codec=t.int32), + database_entry_field("root", 2, codec=_node), + database_entry_field("outcomes", 3, codec=t.list(outcome)), + database_entry_field("part", 4, codec=t.nullable(item)), + database_entry_field("crafted", 5, codec=t.nullable(crafted)), + database_entry_field("missing", 6, codec=_node), ], + name="id", ) @@ -201,9 +239,21 @@ class UnionSurfaceVectorTests(unittest.TestCase): class UnionCodecRuleTests(unittest.TestCase): CANARY = "canary-7f3a91" - def test_nil_union_is_nil_both_ways(self) -> None: - self.assertIsNone(node.encode(None)) - self.assertIsNone(node.decode(None)) + def test_a_union_refuses_nil_both_ways_unless_it_is_wrapped_in_nullable(self) -> None: + for refuse in (lambda: node.encode(None), lambda: node.decode(None)): + with self.assertRaises(ValueError) as caught: + refuse() + self.assertEqual(str(caught.exception), "Union 'Node': expected an arm, found nil") + maybe = t.nullable(node) + self.assertIsNone(maybe.encode(None)) + self.assertIsNone(maybe.decode(None)) + self.assertEqual(maybe.decode(maybe.encode(UnionArm("Text", {"id": "a", "content": "b"}))), UnionArm("Text", {"id": "a", "content": "b"})) + with self.assertRaises(ValueError) as write: + surface_document.encode_payload({**_surface_value(), "outcomes": [None]}) + self.assertEqual( + str(write.exception), + "tests.union_surface slot 3 (outcomes): element 0: Union 'Outcome': expected an arm, found nil", + ) def test_unit_arm_is_written_with_empty_slots_and_reads_back(self) -> None: self.assertEqual(outcome.encode(UnionArm("Denied")), [2, []]) @@ -257,17 +307,26 @@ class UnionCodecRuleTests(unittest.TestCase): UnionArm("Text", {"id": "a", "content": "b"}), ) - def test_many_slot_requires_a_list_and_keeps_nil_elements(self) -> None: + def test_a_list_slot_requires_an_array_and_keeps_nil_elements_of_a_nullable_item(self) -> None: slots = [UnionArm("Text", {"id": "a", "content": "b"}), None] - encoded = use.encode({"id": "u", "composition": None, "slots": slots}) - self.assertEqual(encoded, ["u", None, [[1, ["a", "b"]], None]]) + inner = {"name": "c", "body": None} + encoded = use.encode({"id": "u", "composition": inner, "slots": slots}) + self.assertEqual(encoded, ["u", ["c", None], [[1, ["a", "b"]], None]]) self.assertEqual(use.decode(encoded)["slots"], slots) with self.assertRaises(ValueError) as caught: - use.decode(["u", None, self.CANARY]) - self.assertEqual(str(caught.exception), "DatabaseEntry document type 'Use' slot 2 expected a MessagePack array") + use.decode(["u", ["c", None], self.CANARY]) + self.assertEqual(str(caught.exception), "Use slot 2 (slots): expected an array, found string") + with self.assertRaises(ValueError) as nil_element: + box.decode(["b", [None]]) + self.assertEqual( + str(nil_element.exception), + "Box slot 1 (children): element 0: Union 'Node': expected an arm, found nil", + ) def test_a_list_of_nested_records_keeps_a_nil_element_as_csharp_writes_it(self) -> None: - row = define_record_codec("Row", [("id", 0), database_entry_field("cells", 1, codec=text, many=True)]) + row = define_record_codec( + "Row", [("id", 0), database_entry_field("cells", 1, codec=t.list(t.nullable(text)))] + ) cells = [None, {"id": "c", "content": "x"}, None] encoded = row.encode({"id": "r", "cells": cells}) self.assertEqual(encoded, ["r", [None, ["c", "x"], None]]) @@ -307,22 +366,23 @@ class UnionCodecRuleTests(unittest.TestCase): for arm_name, key in (("Low", -(2**31)), ("Negative", -1), ("High", 2**31 - 1)): self.assertEqual(edge.encode(UnionArm(arm_name)), [key, []]) self.assertEqual(edge.decode([key, []]), UnionArm(arm_name)) - with self.assertRaises(ValueError) as caught: - database_entry_field("loose", 1, many=True) - self.assertEqual(str(caught.exception), "A many DatabaseEntry field requires a codec") - database_entry_field("fine", 1, codec=node, many=True) + for loose in (lambda: node, node.encode, "node", object): + with self.assertRaises(ValueError) as caught: + database_entry_field("loose", 1, codec=loose) # type: ignore[arg-type] + self.assertIn("must be a SlotCodec", str(caught.exception)) + with self.assertRaises(ValueError): + define_union("Loose", {0: ("A", lambda: node)}) # type: ignore[dict-item] class RecordCodecTests(unittest.TestCase): - def test_defaults_fill_a_missing_or_nil_slot_and_a_callable_default_is_called(self) -> None: + def test_a_default_fills_a_slot_past_the_payloads_end_and_a_callable_default_is_called(self) -> None: record = define_record_codec( "Defaulted", [("id", 0), ("label", 1, "none"), ("tags", 2, list), ("plain", 3, 0)], ) self.assertEqual(record.decode(["a"]), {"id": "a", "label": "none", "tags": [], "plain": 0}) - self.assertEqual( - record.decode(["a", None, None, None]), {"id": "a", "label": "none", "tags": [], "plain": 0} - ) + self.assertEqual(record.decode(["a", "x"]), {"id": "a", "label": "x", "tags": [], "plain": 0}) + self.assertEqual(record.decode(["a", None, None, None]), {"id": "a", "label": None, "tags": None, "plain": None}) self.assertEqual(record.decode(["a", "x", ["t"], 5]), {"id": "a", "label": "x", "tags": ["t"], "plain": 5}) def test_a_document_type_names_its_members_and_marks_the_name_and_index_slots(self) -> None: @@ -332,7 +392,7 @@ class RecordCodecTests(unittest.TestCase): name="id", indexes={"by_owner": "owner"}, ) - members = {member.slot: member for member in document.members} + members = {member.slot: member for member in document.catalog_entry().members} self.assertEqual( [members[slot].member_name for slot in (0, 1, 2)], ["Id", "SourceVersion", "Owner"] ) @@ -341,7 +401,7 @@ class RecordCodecTests(unittest.TestCase): def test_a_document_without_a_string_name_marks_no_member_as_the_name(self) -> None: document = define_database_entry_type("tests.named", [("id", 0)], name=lambda value: value["id"]) - self.assertFalse(document.members[0].is_name) + self.assertFalse(document.catalog_entry().members[0].is_name) def test_a_record_that_is_not_an_array_is_refused_naming_the_type_only(self) -> None: for refuse in (lambda r: r.decode({"id": "canary-7f3a91"}), lambda r: r.decode("canary-7f3a91")): @@ -369,20 +429,18 @@ class RecordCodecTests(unittest.TestCase): define_record_codec("Nothing", []) self.assertEqual(str(empty.exception), "DatabaseEntry document types require at least one field") - def test_a_plain_field_helper_makes_a_slot_with_no_codec(self) -> None: + def test_a_plain_field_helper_makes_an_untyped_slot(self) -> None: plain = define_record_codec("Plain", [database_entry_field("id", 0), database_entry_field("label", 1)]) self.assertEqual(plain.decode(plain.encode({"id": "a", "label": "b"})), {"id": "a", "label": "b"}) self.assertEqual(plain.decode(["a"]), {"id": "a", "label": None}) field = database_entry_field("id", 0) - self.assertIsNone(field.codec) - self.assertFalse(field.many) + self.assertIs(field.codec, t.any) - def test_a_directly_constructed_field_requires_a_default_and_is_plain_and_single(self) -> None: + def test_a_directly_constructed_field_requires_a_default_and_is_untyped(self) -> None: with self.assertRaises(TypeError): DatabaseEntryField(name="id", key=0) # type: ignore[call-arg] field = DatabaseEntryField(name="id", key=0, default=dataclasses.MISSING) - self.assertIsNone(field.codec) - self.assertIs(field.many, False) + self.assertIs(field.codec, t.any) def test_slot_codecs_and_fields_are_immutable(self) -> None: with self.assertRaises(dataclasses.FrozenInstanceError): diff --git a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs index 75b078d7..de94c253 100644 --- a/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs +++ b/tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs @@ -2,9 +2,11 @@ using System; using System.Buffers; using System.Buffers.Binary; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Threading.Tasks; using GameCult.Caching.MessagePack; using MessagePack; using NUnit.Framework; @@ -18,12 +20,52 @@ namespace GameCult.Caching.Tests { private const string WriteVariable = "CULTCACHE_WRITE_VECTORS"; private static readonly string[] Leaves = { "int32", "int64", "float32", "float64" }; + private static readonly string[] WideLeaves = { "string", "boolean", "bytes", "any", "list-string", "array-string", "nullable-int32", "list-int32", "record", "list-record", "union" }; [MessagePackObject] public sealed class Int32Slot { [Key(0)] public int V; } [MessagePackObject] public sealed class Int64Slot { [Key(0)] public long V; } [MessagePackObject] public sealed class Float32Slot { [Key(0)] public float V; } [MessagePackObject] public sealed class Float64Slot { [Key(0)] public double V; } + // The wider leaves: one-slot records [value] for each kind of slot a document can declare. + [MessagePackObject] public sealed class StringSlot { [Key(0)] public string? V; } + [MessagePackObject] public sealed class BoolSlot { [Key(0)] public bool V; } + [MessagePackObject] public sealed class BytesSlot { [Key(0)] public byte[]? V; } + [MessagePackObject] public sealed class AnySlot { [Key(0)] public object? V; } + [MessagePackObject] public sealed class ListStringSlot { [Key(0)] public List? V; } + [MessagePackObject] public sealed class ArrayStringSlot { [Key(0)] public string[]? V; } + [MessagePackObject] public sealed class NullableInt32Slot { [Key(0)] public int? V; } + [MessagePackObject] public sealed class ListInt32Slot { [Key(0)] public List? V; } + [MessagePackObject] public sealed class LeafInner { [Key(0)] public string N = string.Empty; [Key(1)] public float W; } + [MessagePackObject] public sealed class RecordSlot { [Key(0)] public LeafInner? V; } + [MessagePackObject] public sealed class ListRecordSlot { [Key(0)] public List? V; } + [Union(0, typeof(LeafCircle))] public interface ILeafShape { } + [MessagePackObject] public sealed class LeafCircle : ILeafShape { [Key(0)] public float R; } + [MessagePackObject] public sealed class UnionSlot { [Key(0)] public ILeafShape? V; } + + // One member of every leaf kind: the record whose catalog rows and payload leaf-record.msgpack pins. + [CultDocument("cultcache.leaf-record", "cultcache.leaf_record.v1")] + [MessagePackObject] + public sealed class LeafRecord + { + [Key(0)] [CultName] public string Id = string.Empty; + [Key(1)] public bool Flag; + [Key(2)] public byte[] Blob = Array.Empty(); + [Key(3)] public int I; + [Key(4)] public long L; + [Key(5)] public float F; + [Key(6)] public double D; + [Key(7)] public int? MaybeI; + [Key(8)] public List Tags = new(); + [Key(9)] public string[] Names = Array.Empty(); + [Key(10)] public object? Any; + [Key(11)] public LeafInner? Nested; + [Key(12)] public List Items = new(); + [Key(13)] public ILeafShape? Shape; + [Key(14)] public float[] Floats = Array.Empty(); + [Key(15)] public float? MaybeF; + } + // A record of every leaf kind, the shape a replicated document has. [MessagePackObject] public sealed class Mixed @@ -39,7 +81,7 @@ namespace GameCult.Caching.Tests private static readonly MessagePackSerializerOptions Options = CultDocumentMessagePackSerialization.OptionsFor(typeof(NumericLeafVectorTests).Assembly); - private static string VectorPath(string name) + internal static string VectorPath(string name) { for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) { @@ -197,6 +239,120 @@ namespace GameCult.Caching.Tests return buffer.WrittenSpan.ToArray(); } + // ---- the wider leaves: reads and one record of every leaf ---- + + // The value token of each wide read case, as MessagePack hex. c1 is a code no reader assigns; d40100 is an application ext, which the any leaf refuses; d6ff00000000, d7ff0000000000000000 and c70cff... are the timestamp32, timestamp64 and timestamp96 exts, which the any leaf reads, while d6fe00000000 and c70100ff are same-shaped non-timestamp exts, which it refuses; 92a161ca3f800000, 92a16105 and 92a161a162 give the record slots a float, an int and a str in the second slot. + private static readonly string[] WideValues = + { + "05", "ff", "cb3ff8000000000000", "ca3fc00000", "c0", "c2", "c3", "a161", "d90161", "c40100", "90", "9101", "91a161", "80", "81a161a162", + "920091ca3f800000", "9291a161ca3f800000", "9192a161ca3f800000", "c1", + "92a161ca3f800000", "92a16105", "92a161a162", "d40100", + "d6ff00000000", "d7ff0000000000000000", "c70cff000000000000000000000000", "d6fe00000000", "c70100ff", + }; + + // What the reference reads from a one-slot record of the leaf, and the bytes it writes back for what it read. + private static (bool Accepted, byte[] Rewrite) ReadWide(string leaf, byte[] payload) => leaf switch + { + "string" => Reread(payload), + "boolean" => Reread(payload), + "bytes" => Reread(payload), + "any" => Reread(payload), + "list-string" => Reread(payload), + "array-string" => Reread(payload), + "nullable-int32" => Reread(payload), + "list-int32" => Reread(payload), + "record" => Reread(payload), + "list-record" => Reread(payload), + _ => Reread(payload), + }; + + private static (bool Accepted, byte[] Rewrite) Reread(byte[] payload) + { + T value; + try + { + value = MessagePackSerializer.Deserialize(payload, Options); + } + catch (Exception) + { + return (false, Array.Empty()); + } + + return (true, MessagePackSerializer.Serialize(value, Options)); + } + + // leaves-reads.msgpack: an array of [leaf, payload (bin), accepted, rewrite (bin: the reference's bytes for what it read, empty when refused)]. + private static byte[] LeavesReadsVector() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(WideLeaves.Length * WideValues.Length); + foreach (var leaf in WideLeaves) + foreach (var value in WideValues) + { + var payload = Payload(value); + var (accepted, rewrite) = ReadWide(leaf, payload); + writer.WriteArrayHeader(4); + writer.Write(leaf); + writer.Write(payload); + writer.Write(accepted); + writer.Write(rewrite); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + // Every float32 slot holds 0.1f, which float32 holds only as its own rounding, and the double slot 0.1, which float32 cannot hold; 1.5f is exact. + private static LeafRecord Everything() => new() + { + Id = "leaves", Flag = true, Blob = new byte[] { 1, 2, 3 }, I = -7, L = 5_000_000_000, F = 0.1f, D = 0.1, MaybeI = 9, + Tags = new List { "a", "b" }, Names = new[] { "x", "y" }, Any = "text", + Nested = new LeafInner { N = "inner", W = 0.1f }, Items = new List { new() { N = "one", W = 0.1f }, new() { N = "two", W = 1.5f } }, + Shape = new LeafCircle { R = 0.1f }, Floats = new[] { 0.1f, 1.5f }, MaybeF = 0.1f, + }; + + // leaf-record.msgpack: [members, payload]. members are the catalog rows the store writes for LeafRecord, each + // [slot, memberName, typeName, isReference, isMany, targetSchemaName, isName, indexAlias]; payload (bin) is the record's bytes in that store. + private static async Task LeafRecordVector() + { + var path = Path.Combine(Path.GetTempPath(), $"cultlib-leaf-record-{Guid.NewGuid():N}.cc"); + try + { + var store = new SingleFileMessagePackBackingStore(path); + var cache = new CultCache(); + cache.AddBackingStore(store); + await cache.AddAsync(Everything()); + store.PushAll(); + var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)); + var members = snapshot.SchemaCatalog.Single(entry => entry.SchemaName == "cultcache.leaf-record").Members; + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(2); + writer.WriteArrayHeader(members.Length); + foreach (var member in members) + { + writer.WriteArrayHeader(8); + writer.Write(member.Slot); + writer.Write(member.MemberName); + writer.Write(member.TypeName); + writer.Write(member.IsReference); + writer.Write(member.IsMany); + writer.Write(member.TargetSchemaName); + writer.Write(member.IsName); + writer.Write(member.IndexAlias); + } + + writer.Write(snapshot.Records.Single().Payload); + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + // ---- tests ---- [Test] @@ -229,6 +385,30 @@ namespace GameCult.Caching.Tests Assert.That((read.Id, read.I, read.L, read.F, read.D), Is.EqualTo(("n1", 7, 8L, 1.5f, 2.5))); } + [Test] + public void LeavesReadsVectorMatches() => Check("leaves-reads.msgpack", LeavesReadsVector()); + + [Test] + public async Task LeafRecordVectorMatches() + { + var written = await LeafRecordVector(); + Check("leaf-record.msgpack", written); + var reader = new MessagePackReader(written); + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2)); + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(16), "one member per leaf"); + } + + [Test] + public void LeavesReadCoverEveryLeafAndBothOutcomes() + { + var rows = WideLeaves.SelectMany(leaf => WideValues.Select(value => (leaf, result: ReadWide(leaf, Payload(value))))).ToList(); + foreach (var leaf in WideLeaves) + { + Assert.That(rows.Count(row => row.leaf == leaf && row.result.Accepted), Is.GreaterThan(0), $"{leaf} accepts"); + Assert.That(rows.Count(row => row.leaf == leaf && !row.result.Accepted), Is.GreaterThan(0), $"{leaf} refuses"); + } + } + [Test] public void ReadsCoverEveryLeafAndBothOutcomes() { diff --git a/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs new file mode 100644 index 00000000..6b26e90b --- /dev/null +++ b/tests/GameCult.Caching.Tests/PythonWrittenStoreTests.cs @@ -0,0 +1,93 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching.MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // The reference opens a store a Python typed-leaf definition (cultcache-py) wrote. The stores are vectors written only + // by cultcache-py's PythonWrittenStoreVectorTests (CULTCACHE_WRITE_VECTORS=1), values authored in Python. + public class PythonWrittenStoreTests + { + private static string Store(string name, string scratch) + { + var copy = Path.Combine(scratch, name); + File.Copy(NumericLeafVectorTests.VectorPath(Path.Combine("py-store", name)), copy, true); + return copy; + } + + [Test] + public async Task APythonWrittenTypedLeafStoreOpensWithEverySlotItsCSharpValue() + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var cache = new CultCache(); + cache.AddBackingStore(new SingleFileMessagePackBackingStore(Store("leaf-record.cc", scratch))); + await cache.PullAllBackingStoresAsync(); + var doc = cache.AllEntries.OfType().Single(); + static int Bits(float f) => BitConverter.SingleToInt32Bits(f); + Assert.Multiple(() => + { + Assert.That(doc.Id, Is.EqualTo("py")); + Assert.That(doc.Flag, Is.False); + Assert.That(doc.Blob, Is.EqualTo(new byte[] { 1, 0xff })); + Assert.That(doc.I, Is.EqualTo(int.MinValue)); + Assert.That(doc.L, Is.EqualTo(long.MaxValue)); + Assert.That(Bits(doc.F), Is.EqualTo(Bits(0.1f))); + Assert.That(doc.D, Is.EqualTo(1e-300)); + Assert.That(doc.MaybeI, Is.Null); + Assert.That(doc.Tags, Is.Empty); + Assert.That(doc.Names, Is.EqualTo(new[] { "é", "z" })); + var any = (IDictionary)doc.Any!; + Assert.That(Convert.ToInt64(any["n"]), Is.EqualTo(-3)); + var k = (object[])any["k"]; + Assert.That(Convert.ToInt64(k[0]), Is.EqualTo(1)); + Assert.That(k[1], Is.EqualTo(true)); + Assert.That(k[2], Is.Null); + Assert.That(k[3], Is.EqualTo(2.5)); + Assert.That(k[4], Is.EqualTo("s")); + Assert.That(doc.Nested, Is.Null); + Assert.That(doc.Items.Select(i => i.N), Is.EqualTo(new[] { "one", "two" })); + Assert.That(Bits(doc.Items[0].W), Is.EqualTo(Bits((float)(9007199254740992L + 536870913L)))); + Assert.That(Bits(doc.Items[1].W), Is.EqualTo(Bits(1e-40f))); + Assert.That(Bits(((NumericLeafVectorTests.LeafCircle)doc.Shape!).R), Is.EqualTo(Bits(0.1f))); + Assert.That(doc.Floats.Select(Bits), Is.EqualTo(new[] { 0.1f, 3f, -0.0f, float.PositiveInfinity }.Select(Bits))); + Assert.That(doc.MaybeF, Is.Null); + }); + } + finally + { + Directory.Delete(scratch, true); + } + } + + [TestCase(3)] + [TestCase(5)] + [TestCase(7)] + [TestCase(14)] + public async Task APythonWrittenCatalogNamingAnotherTypeForASlotIsRefused(int slot) + { + var scratch = Directory.CreateTempSubdirectory().FullName; + try + { + var cache = new CultCache(); + var path = Store($"wrong-slot-{slot}.cc", scratch); + var error = Assert.CatchAsync(async () => + { + cache.AddBackingStore(new SingleFileMessagePackBackingStore(path)); + await cache.PullAllBackingStoresAsync(); + }); + Assert.That(error!.Message, Does.Contain($"slot {slot} changed type")); + } + finally + { + Directory.Delete(scratch, true); + } + } + } +} diff --git a/tests/vectors/numeric-leaves/README.md b/tests/vectors/numeric-leaves/README.md index 3007204e..78302cbc 100644 --- a/tests/vectors/numeric-leaves/README.md +++ b/tests/vectors/numeric-leaves/README.md @@ -1,11 +1,14 @@ # numeric-leaves vectors -Written only by the C# reference, `tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs`, with +Except `py-store/` (below), written only by the C# reference, `tests/GameCult.Caching.Tests/NumericLeafVectorTests.cs`, with `CULTCACHE_WRITE_VECTORS=1`. Every other run reads these files and compares them byte for byte; a changed vector is a reviewed commit. Other runtimes decode them and are held to the same answers. -A document with one slot, `[value]`, whose slot is a C# `int`, `long`, `float` or `double` (leaf `int32`, `int64`, -`float32`, `float64`). +These vectors hold every leaf kind, not only numbers. A document with one slot, `[value]`, whose slot is a C# `int`, +`long`, `float` or `double` (leaf `int32`, `int64`, `float32`, `float64`), or a `string`, `bool`, `byte[]`, `object`, +`List`, `string[]`, `int?`, `List`, a two-slot record class, a `List` of it or a `[Union]` interface (leaf +`string`, `boolean`, `bytes`, `any`, `list-string`, `array-string`, `nullable-int32`, `list-int32`, `record`, `list-record`, +`union`). - `reads.msgpack`: an array of `[leaf, payload, accepted, bits]`. `payload` (bin) is the one-slot record fed to the reference; `accepted` is whether it read; `bits` (bin, 8 bytes, big-endian) is the int64 it read, or the IEEE bits of @@ -14,3 +17,23 @@ A document with one slot, `[value]`, whose slot is a C# `int`, `long`, `float` o bits in hex for a float leaf; `bytes` (bin) is the one-slot record the reference writes for that value. - `float32-nan.msgpack`: an array of `[payload, bytes]`. `payload` (bin) is a one-slot `float` record holding an infinity or a NaN; `bytes` (bin) is what the reference writes after reading it. It keeps every bit, a signalling NaN's included. +- `leaves-reads.msgpack`: an array of `[leaf, payload, accepted, rewrite]` for the wider leaves. `payload` (bin) is the + one-slot record `[token]` fed to the reference; `accepted` is whether it read; `rewrite` (bin) is what the reference + writes after reading it, and empty when refused. Every token is fed to every leaf: 11 leaves x 28 tokens, 308 rows. + The tokens include an application ext (`d40100`, which the `any` leaf refuses), the timestamp exts (`d6ff00000000`, + `d7ff0000000000000000`, `c70cff000000000000000000000000`, which the `any` leaf reads and rewrites as `91d6ff00000000`), + same-shaped non-timestamp exts (`d6fe00000000`, `c70100ff`, which it refuses), and complete, int-in-float-slot and + str-in-float-slot two-slot records. + On the `any` leaf's rewrite rows a sibling is held to the rewrite's value kind (nil, bool, int, float, str, bin, array, map; bool, + int and float are distinct, so `true`, `1` and `1.0` never match) and its exact value, recursively, not to its MessagePack numeric + width (the reference widens a boxed fixint, so the record `9105` rewrites as `91cc05`, and a float 32 as a float 64; ruling + `cultlib-gaps:ruling:any-rewrite-binds-kind-and-value`). + Every other leaf's rewrite binds bytes. +- `leaf-record.msgpack`: `[members, payload]` for one `cultcache.leaf-record` document holding every leaf. `members` are + the catalog rows the single-file store writes, each `[slot, memberName, typeName, isReference, isMany, + targetSchemaName, isName, indexAlias]`; `payload` (bin) is the record's bytes in that store. + +- `py-store/`: single-file stores written only by cultcache-py (`PythonWrittenStoreVectorTests` in `test_typed_leaves.py`, with + `CULTCACHE_WRITE_VECTORS=1`), never by C#. `leaf-record.cc` holds one `cultcache.leaf-record` document authored in Python; the + reference (`PythonWrittenStoreTests`) opens it and asserts every field. `wrong-slot-N.cc` is the same store whose catalog names + another C# type for slot N (3, 5, 7, 14); the reference refuses each with `slot N changed type`. diff --git a/tests/vectors/numeric-leaves/leaf-record.msgpack b/tests/vectors/numeric-leaves/leaf-record.msgpack new file mode 100644 index 00000000..224521a2 Binary files /dev/null and b/tests/vectors/numeric-leaves/leaf-record.msgpack differ diff --git a/tests/vectors/numeric-leaves/leaves-reads.msgpack b/tests/vectors/numeric-leaves/leaves-reads.msgpack new file mode 100644 index 00000000..0ea61555 Binary files /dev/null and b/tests/vectors/numeric-leaves/leaves-reads.msgpack differ diff --git a/tests/vectors/numeric-leaves/py-store/leaf-record.cc b/tests/vectors/numeric-leaves/py-store/leaf-record.cc new file mode 100644 index 00000000..7f796df6 Binary files /dev/null and b/tests/vectors/numeric-leaves/py-store/leaf-record.cc differ diff --git a/tests/vectors/numeric-leaves/py-store/wrong-slot-14.cc b/tests/vectors/numeric-leaves/py-store/wrong-slot-14.cc new file mode 100644 index 00000000..743722d3 Binary files /dev/null and b/tests/vectors/numeric-leaves/py-store/wrong-slot-14.cc differ diff --git a/tests/vectors/numeric-leaves/py-store/wrong-slot-3.cc b/tests/vectors/numeric-leaves/py-store/wrong-slot-3.cc new file mode 100644 index 00000000..492352a5 Binary files /dev/null and b/tests/vectors/numeric-leaves/py-store/wrong-slot-3.cc differ diff --git a/tests/vectors/numeric-leaves/py-store/wrong-slot-5.cc b/tests/vectors/numeric-leaves/py-store/wrong-slot-5.cc new file mode 100644 index 00000000..799f59a8 Binary files /dev/null and b/tests/vectors/numeric-leaves/py-store/wrong-slot-5.cc differ diff --git a/tests/vectors/numeric-leaves/py-store/wrong-slot-7.cc b/tests/vectors/numeric-leaves/py-store/wrong-slot-7.cc new file mode 100644 index 00000000..f1335f4e Binary files /dev/null and b/tests/vectors/numeric-leaves/py-store/wrong-slot-7.cc differ