From 13ce5ea50a0e231819eb6f33ae76db1296ac06d2 Mon Sep 17 00:00:00 2001 From: Libby Date: Thu, 12 Nov 2026 05:23:25 +0000 Subject: [PATCH] Python: a payload is one MessagePack value (bytes-after-py) (#129) --- packages/cultcache-py/CHANGELOG.md | 6 +- packages/cultcache-py/README.md | 1 + packages/cultcache-py/pyproject.toml | 2 +- .../cultcache-py/src/cultcache_py/__init__.py | 2 + .../src/cultcache_py/documents.py | 26 ++++- .../cultcache-py/tests/test_union_parity.py | 72 ++++++++++++++ packages/cultmesh-py/pyproject.toml | 2 +- packages/cultnet-py/CHANGELOG.md | 1 + packages/cultnet-py/pyproject.toml | 2 +- .../cultnet-py/src/cultnet_py/messages.py | 3 +- .../cultnet-py/src/cultnet_py/replication.py | 6 +- packages/cultnet-py/tests/test_cultnet.py | 45 +++++++++ .../tests/test_declared_dependency_ranges.py | 94 +++++++++++++++++++ 13 files changed, 251 insertions(+), 11 deletions(-) create mode 100644 packages/cultnet-py/tests/test_declared_dependency_ranges.py diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index 08fb4d1f..ea3f72fd 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Added + +- `read_payload_tree(type_name, payload, decoder=None)`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged. + ### Changed - `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import. @@ -10,7 +14,7 @@ ### Breaking - A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`. - +- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` ` payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#. - A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it. - Added `foreign_records()` and the frozen `ForeignRecord(key, schema_id, schema_name, stored_at)`, sorted by key in UTF-16 code-unit order. `stores.retyped` and `stores.written_owns` lose their underscore: the cache judge and the single-file store's commit guard ask one predicate. diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 72d51562..4a7236fd 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -19,6 +19,7 @@ discovery, sessions, local server) build on it, mirroring `cultcache-rs` / `[format, schemaCatalog, records]` - records store raw MessagePack payload bytes under schema-catalog identity - payloads decode only through the registered document definition for that type +- a payload is exactly one MessagePack value: bytes after it are refused on read, by count - a cache holds one record per key whatever its type: a write of a second type at a held key is refused with `SchemaConflictError` (in memory too), and two stores holding one key refuse the load - a store record no registered type claims loads and is carried as foreign: `cache.foreign_records()` lists it, the store diff --git a/packages/cultcache-py/pyproject.toml b/packages/cultcache-py/pyproject.toml index 6c46d102..15b724ba 100644 --- a/packages/cultcache-py/pyproject.toml +++ b/packages/cultcache-py/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cultcache-py" -version = "0.3.0" +version = "0.4.0" description = "Python CultCache runtime for GameCult typed state." readme = "README.md" requires-python = ">=3.10" diff --git a/packages/cultcache-py/src/cultcache_py/__init__.py b/packages/cultcache-py/src/cultcache_py/__init__.py index 4873b6ed..69828bf3 100644 --- a/packages/cultcache-py/src/cultcache_py/__init__.py +++ b/packages/cultcache-py/src/cultcache_py/__init__.py @@ -16,6 +16,7 @@ from .documents import ( define_document_type, define_record_codec, define_union, + read_payload_tree, ) from .leaves import SlotCodec from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError @@ -40,6 +41,7 @@ __all__ = [ "StoreUnreadableError", "UnionArm", "UnreadSlots", + "read_payload_tree", "leaves", "database_entry_field", "define_database_entry_type", diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 4f7dfe7d..ff9039dc 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -158,6 +158,26 @@ def _messagepack_payload_decoder(payload: bytes) -> Any: return msgpack.unpackb(payload, raw=False) +def read_payload_tree(type_name: str, payload: bytes, decoder: Callable[[bytes], Any] | None = None) -> Any: + """The one read of a payload's tree: exactly one MessagePack value, within the nesting limit. Depth is judged first, + on the value read so far, so a tail never pre-empts it (C# throws at container 257 before it reaches the tail); then a + tail is refused by count and never echoed. Raised outside the handler so no `__context__` keeps the tail.""" + after = -1 + unpacked: Any = None + try: + tree = (decoder or _messagepack_payload_decoder)(payload) + except _STACK_FAULTS: + raise CultCachePayloadDepthError(type_name, "read") from None + except msgpack.ExtraData as error: + after = len(error.extra) + unpacked = error.unpacked + if after >= 0: + _refuse_past_depth_limit(unpacked, type_name, "read") + raise ValueError(f"{type_name} payload has {after} bytes after its first MessagePack value (refused on read).") + _refuse_past_depth_limit(tree, type_name, "read") + return tree + + @dataclass(frozen=True) class DatabaseEntryField: name: str @@ -234,11 +254,7 @@ class DocumentDefinition(Generic[T]): def decode_payload_marked(self, payload: bytes) -> tuple[T, UnreadSlots | None]: """The value and what the decode found past the declared slots, at any depth. The payload depth check runs first: a payload past the limit is refused, never marked.""" - try: - tree = self.payload_decoder(payload) - except _STACK_FAULTS: - raise CultCachePayloadDepthError(self.type, "read") from None - _refuse_past_depth_limit(tree, self.type, "read") + tree = read_payload_tree(self.type, payload, self.payload_decoder) outer, opened = _current_scope(), _SlotScope() _scope.open = opened try: diff --git a/packages/cultcache-py/tests/test_union_parity.py b/packages/cultcache-py/tests/test_union_parity.py index 1e19ce8d..793933ff 100644 --- a/packages/cultcache-py/tests/test_union_parity.py +++ b/packages/cultcache-py/tests/test_union_parity.py @@ -661,6 +661,78 @@ def _as_tuple(tree: object) -> object: return tuple(_as_tuple(child) for child in tree) if isinstance(tree, list) else tree + +class BytesAfterVectorTests(unittest.TestCase): + CLAUSE = "bytes after its first MessagePack value (refused on read)" + + def _assert_chain_clean(self, error: BaseException, file: str) -> None: + link: BaseException | None = error + while link is not None: + self.assertNotIn(CANARY, str(link), file) + self.assertNotIsInstance(link, msgpack.ExtraData, file) + link = link.__cause__ or link.__context__ + + def test_every_bytes_after_vector_is_refused(self) -> None: + base = _vector("union-surface.msgpack") + entries = msgpack.unpackb(_vector("bytes-after-outcomes.msgpack"), raw=False) + self.assertEqual(len(entries), 4) + for file, outcome_kind, union, key, detail in entries: + self.assertEqual((outcome_kind, union, key, detail), ("refused", "UnionSurface", None, "bytes-after"), file) + payload = _vector(file) + self.assertTrue(payload.startswith(base), file) + count = len(payload) - len(base) + with self.assertRaises(ValueError, msg=file) as caught: + surface_document.decode_payload(payload) + error = caught.exception + self.assertIs(type(error), ValueError, file) + self.assertNotIsInstance(error, CultCachePayloadDepthError, file) + self.assertEqual( + str(error), f"{surface_document.type} payload has {count} {self.CLAUSE}.", file + ) + self.assertNotIn("the limit is", str(error), file) + self._assert_chain_clean(error, file) + + def test_a_single_file_store_holding_a_tail_as_a_record_payload_refuses_on_load(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "tail.cc" + shutil.copyfile(VECTORS / "union-surface.cc", path) + store = SingleFileMessagePackBackingStore(path) + [envelope] = store.pull_all() + store.push(dataclasses.replace(envelope, payload=_vector("bytes-after-nil.msgpack"))) + cache = ( + CultCache.builder() + .register_document_type(surface_document) + .add_generic_store(SingleFileMessagePackBackingStore(path)) + .build() + ) + with self.assertRaises(ValueError) as caught: + cache.pull_all_backing_stores() + self.assertIn(f"1 {self.CLAUSE}", str(caught.exception)) + self.assertIn(surface_document.type, str(caught.exception)) + self.assertIsNone(cache.get(surface_document, "surface")) + + def test_depth_wins_over_a_tail_whatever_the_unpackers_own_stack_limit(self) -> None: + # Vector-independent: a value nested past 256 is the depth refusal even with a tail, as C# throws at container 257 + # before it reaches the tail; 2000 is past msgpack's own stack, so the winner is not a library constant. + for depth in (257, 300, 900, 2000, 100000): + for tail in (b"\xc0", b"\xc1\xc1"): + with self.subTest(depth=depth, tail=len(tail)): + with self.assertRaises(CultCachePayloadDepthError) as caught: + surface_document.decode_payload(b"\x91" * depth + b"\xc0" + tail) + self.assertIn("refused on read", str(caught.exception)) + for depth in (1, 255, 256): + with self.subTest(at=depth): + with self.assertRaises(ValueError) as caught: + surface_document.decode_payload(b"\x91" * depth + b"\xc0\xc0") + self.assertNotIsInstance(caught.exception, CultCachePayloadDepthError) + self.assertIn(f"1 {self.CLAUSE}", str(caught.exception)) + + def test_a_payload_that_ends_early_is_the_decoders_not_bytes_after(self) -> None: + truncated = _vector("union-surface.msgpack")[:-1] + with self.assertRaises(Exception) as caught: + surface_document.decode_payload(truncated) + self.assertNotIn("bytes after", str(caught.exception)) + class PayloadDepthRuleTests(unittest.TestCase): """The nesting rule at its seam, on definitions that are not unions: the same routine serves any payload codec.""" diff --git a/packages/cultmesh-py/pyproject.toml b/packages/cultmesh-py/pyproject.toml index 5f9bda4d..18f23060 100644 --- a/packages/cultmesh-py/pyproject.toml +++ b/packages/cultmesh-py/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0", "cryptography>=41"] +dependencies = ["cultcache-py>=0.4.0", "cultnet-py>=0.1.0", "cryptography>=41"] [project.scripts] cultmesh-py-daemon = "cultmesh_py.daemon:main" cultmesh-py-verify = "cultmesh_py.verify:main" diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index b8a3773e..525bd4bd 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). A raw record with bytes after its first value gives no schema hint to `resolve_document_for_raw_record`, so an unknown schema id stays `KeyError` rather than being routed; a registered one is refused by count when applied. - `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out. - `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move. - Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client. diff --git a/packages/cultnet-py/pyproject.toml b/packages/cultnet-py/pyproject.toml index 41c4e8f6..09db2733 100644 --- a/packages/cultnet-py/pyproject.toml +++ b/packages/cultnet-py/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["cultcache-py>=0.2.0", "msgpack>=1.0"] +dependencies = ["cultcache-py>=0.4.0", "msgpack>=1.0"] [project.optional-dependencies] crypto = ["cryptography>=42"] diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index d5e26b7a..c9b7d4a6 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import Any import msgpack +from cultcache_py import read_payload_tree from .unicode_messageid_tables import DEFAULT_IGNORABLE, LETTER_OR_NUMBER @@ -544,7 +545,7 @@ def encode_witness_artifact_bundle_payload(bundle: dict[str, Any] | CultNetWitne def decode_witness_artifact_bundle_payload(payload: bytes) -> dict[str, Any]: - slots = msgpack.unpackb(payload, raw=False) + slots = read_payload_tree("cultnet.witness_artifact_bundle", payload) if not isinstance(slots, list) or len(slots) < 8: raise ValueError("CultNet witness artifact bundle payload must be an 8-slot MessagePack array") return { diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 531c0012..28f5da33 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -171,9 +171,13 @@ def resolve_document_for_raw_record( def _infer_schema_version_from_payload(payload: bytes) -> str | None: + import msgpack # type: ignore try: - import msgpack # type: ignore decoded = msgpack.unpackb(payload, raw=False) + except msgpack.ExtraData: + # A payload with bytes after its value gives no routing hint: the record is unroutable by inference, and a + # registered schema id routes it to a definition whose decode refuses the tail by count. + return None except Exception: return None if isinstance(decoded, list) and decoded and isinstance(decoded[0], str): diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 48972221..cdfaee5b 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -1971,6 +1971,34 @@ class CultNetTests(unittest.TestCase): apply_raw_document_record(cache, schema_document_map([first, second]), raw(first)) # the same type still replaces self.assertEqual(cache.get_required(first, "shared").name, "remote") + def test_a_raw_record_with_a_tail_is_unroutable_by_inference_and_refused_by_count_when_routed(self) -> None: + import msgpack # type: ignore + from cultcache_py import define_document_type + from cultnet_py import resolve_document_for_raw_record + + document = define_document_type("tests.tail_note", schema_id="tests.tail_note", schema_version="tests.tail_note.v1", + payload_encoder=msgpack.packb, payload_decoder=lambda payload: msgpack.unpackb(payload, raw=False)) + known = document.catalog_entry().schema_id + head = msgpack.packb(["tests.tail_note.v1", "x"], use_bin_type=True) + + def raw(schema_id: str, payload: bytes) -> dict: + return {"schemaId": schema_id, "recordKey": "k", "storedAt": "2026-06-13T00:00:00Z", + "payloadEncoding": "messagepack", "payload": payload} + + documents = schema_document_map([document]) + # Routing by inference reads a hint, not a payload: a payload with a tail gives no hint, so an unknown schema id + # stays the unknown-schema error (KeyError) even when the first value names a registered schema. + self.assertIs(resolve_document_for_raw_record(documents, "unknown", raw("unknown", head)), document) + with self.assertRaises(KeyError): + resolve_document_for_raw_record(documents, "unknown", raw("unknown", head + b"\xc0")) + # A registered schema id routes, and the tail is refused by count when the record is applied. + cache = CultCache() + cache.register_document_type(document) + with self.assertRaises(ValueError) as caught: + apply_raw_document_record(cache, documents, raw(known, head + b"\xc0")) + self.assertIn("1 bytes after its first MessagePack value", str(caught.exception)) + self.assertIsNone(cache.get(document, "k")) + def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None: document = define_database_entry_type( "replica.item", @@ -2661,6 +2689,23 @@ class CultNetTests(unittest.TestCase): self.assertEqual(responses[0]["details"]["value"], schema_version) self.assertNotIn("messageId", responses[0]) + def test_witness_artifact_bundle_payload_is_one_value_and_depth_wins_over_a_tail(self) -> None: + import msgpack # type: ignore + from cultcache_py import CultCachePayloadDepthError + + payload = msgpack.packb(["b", "k", "t", {}, [], [], [], {}], use_bin_type=True) + decode_witness_artifact_bundle_payload(payload) + for tail, count in ((b"\xc0", 1), (b"\xc1\xc1", 2)): + with self.assertRaises(ValueError) as caught: + decode_witness_artifact_bundle_payload(payload + tail) + self.assertIs(type(caught.exception), ValueError) + self.assertEqual( + str(caught.exception), + f"cultnet.witness_artifact_bundle payload has {count} bytes after its first MessagePack value (refused on read).", + ) + with self.assertRaises(CultCachePayloadDepthError): + decode_witness_artifact_bundle_payload(b"\x91" * 300 + b"\xc0\xc0") + def test_cultnet_simulation_consensus_dedupes_witnesses_and_requires_quorum(self) -> None: from cultnet_py import CultNetSimulationConsensus diff --git a/packages/cultnet-py/tests/test_declared_dependency_ranges.py b/packages/cultnet-py/tests/test_declared_dependency_ranges.py new file mode 100644 index 00000000..9baf15fc --- /dev/null +++ b/packages/cultnet-py/tests/test_declared_dependency_ranges.py @@ -0,0 +1,94 @@ +"""A sibling package resolves through the workspace's editable install, so a package that starts importing a new name from +cultcache-py passes whatever range it declares. A consumer outside the workspace installs the newest release the range +admits. This is the Python form of scripts/test-declared-dependency-ranges.mjs: for every tagged cultcache-py release +(`cultcache-py-vX.Y.Z`) that the `cultcache-py>=` bound of cultnet-py or cultmesh-py admits, every name those packages +import from cultcache_py must be defined by that release's source. It needs git with the release tags and skips without +them. Only `>=` bounds are read; star imports and names reached by attribute access are not checked.""" + +from __future__ import annotations + +import ast +import re +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +PACKAGES = ROOT / "packages" +CONSUMERS = ("cultnet-py", "cultmesh-py") + + +def _version(text: str) -> tuple[int, ...]: + return tuple(int(part) for part in text.split(".")) + + +def _git(*args: str) -> str | None: + try: + done = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True) + except (OSError, subprocess.CalledProcessError): + return None + return done.stdout + + +def _lower_bound(package: str) -> tuple[int, ...]: + manifest = (PACKAGES / package / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'"cultcache-py>=(\d+\.\d+\.\d+)"', manifest) + assert match, f"{package} declares no cultcache-py>= bound" + return _version(match.group(1)) + + +def _imports(package: str) -> dict[str, set[str]]: + """module -> names imported from it, for every `from cultcache_py[.x] import names` under the package's src.""" + found: dict[str, set[str]] = {} + for source in (PACKAGES / package / "src").rglob("*.py"): + for node in ast.walk(ast.parse(source.read_text(encoding="utf-8-sig"))): + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + if node.module == "cultcache_py" or node.module.startswith("cultcache_py."): + found.setdefault(node.module, set()).update(alias.name for alias in node.names if alias.name != "*") + return found + + +def _defined(tag: str, module: str) -> set[str] | None: + base = "packages/cultcache-py/src/" + module.replace(".", "/") + for path in (base + ".py", base + "/__init__.py"): + text = _git("show", f"{tag}:{path}") + if text is None: + continue + names: set[str] = set() + for node in ast.parse(text.lstrip("\ufeff")).body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update((alias.asname or alias.name).split(".")[0] for alias in node.names) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + names.add(target.id) + return names + return None + + +class DeclaredDependencyRangeTests(unittest.TestCase): + def test_every_admitted_cultcache_release_defines_what_the_consumers_import(self) -> None: + listed = _git("tag", "--list", "cultcache-py-v*") + tags = sorted((t for t in (listed or "").split() if re.fullmatch(r"cultcache-py-v\d+\.\d+\.\d+", t)), + key=lambda t: _version(t.rsplit("v", 1)[1])) + if not tags: + self.skipTest("git with the cultcache-py release tags is not available") + current = re.search(r'^version = "(\d+\.\d+\.\d+)"', (PACKAGES / "cultcache-py" / "pyproject.toml").read_text(encoding="utf-8"), re.M) + assert current + for package in CONSUMERS: + bound = _lower_bound(package) + self.assertLessEqual(bound, _version(current.group(1)), f"{package}'s bound is above the workspace cultcache-py") + imports = _imports(package) + for tag in tags: + if _version(tag.rsplit("v", 1)[1]) < bound: + continue + for module, names in imports.items(): + defined = _defined(tag, module) + self.assertIsNotNone(defined, f"{package} imports {module}, which {tag} does not have") + self.assertEqual(sorted(names - defined), [], f"{package} imports names {tag} lacks from {module}") + + +if __name__ == "__main__": + unittest.main()