diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index 525bd4bd..b7e73ae3 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased (breaking; version 0.x, so the next minor) -- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). A raw record with bytes after its first value gives no schema hint to `resolve_document_for_raw_record`, so an unknown schema id stays `KeyError` rather than being routed; a registered one is refused by count when applied. +- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). `resolve_document_for_raw_record` reads the schema hint from a raw record's first value whether or not bytes follow it, so a record routed by its stamp is refused by count when applied (`ValueError`, in a snapshot, a shard log and a direct apply) like one routed by a registered schema id; `apply_shard_log_response` used to skip it silently as an unknown schema. A tail on a payload whose first value names no registered schema is still `KeyError`, which a shard log skips. - `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out. - `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move. - Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client. diff --git a/packages/cultnet-py/src/cultnet_py/replication.py b/packages/cultnet-py/src/cultnet_py/replication.py index 28f5da33..345f1c62 100644 --- a/packages/cultnet-py/src/cultnet_py/replication.py +++ b/packages/cultnet-py/src/cultnet_py/replication.py @@ -174,10 +174,11 @@ def _infer_schema_version_from_payload(payload: bytes) -> str | None: import msgpack # type: ignore try: decoded = msgpack.unpackb(payload, raw=False) - except msgpack.ExtraData: - # A payload with bytes after its value gives no routing hint: the record is unroutable by inference, and a - # registered schema id routes it to a definition whose decode refuses the tail by count. - return None + except msgpack.ExtraData as extra: + # The hint is read from the first value; the tail is not this function's to judge. The routed definition refuses + # it by count when the record is applied, as it does for a record routed by a registered schema id. Treating a + # tail as "no hint" made a stamp-routed record look like an unknown schema, which a shard log skips silently. + decoded = extra.unpacked except Exception: return None if isinstance(decoded, list) and decoded and isinstance(decoded[0], str): diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index cdfaee5b..73b41a79 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -1971,7 +1971,7 @@ class CultNetTests(unittest.TestCase): apply_raw_document_record(cache, schema_document_map([first, second]), raw(first)) # the same type still replaces self.assertEqual(cache.get_required(first, "shared").name, "remote") - def test_a_raw_record_with_a_tail_is_unroutable_by_inference_and_refused_by_count_when_routed(self) -> None: + def test_a_raw_record_with_a_tail_is_routed_by_its_stamp_and_refused_by_count_on_every_apply_path(self) -> None: import msgpack # type: ignore from cultcache_py import define_document_type from cultnet_py import resolve_document_for_raw_record @@ -1986,18 +1986,33 @@ class CultNetTests(unittest.TestCase): "payloadEncoding": "messagepack", "payload": payload} documents = schema_document_map([document]) - # Routing by inference reads a hint, not a payload: a payload with a tail gives no hint, so an unknown schema id - # stays the unknown-schema error (KeyError) even when the first value names a registered schema. - self.assertIs(resolve_document_for_raw_record(documents, "unknown", raw("unknown", head)), document) + # The stamp is read from the first value whether or not a tail follows, so an unknown schema id whose payload names + # a registered schema routes to it; the tail is then refused by count where the record is applied. + self.assertIs(resolve_document_for_raw_record(documents, "unknown", raw("unknown", head + b"\xc0")), document) + for schema_id in (known, "unknown"): + cache = CultCache() + cache.register_document_type(document) + with self.assertRaises(ValueError, msg=schema_id) as caught: + apply_raw_document_record(cache, documents, raw(schema_id, head + b"\xc0")) + self.assertIn("1 bytes after its first MessagePack value", str(caught.exception)) + self.assertIsNone(cache.get(document, "k")) + snapshot = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": "s", + "documents": [raw(schema_id, head + b"\xc0")]} + with self.assertRaises(ValueError, msg=schema_id): + apply_raw_snapshot(cache, [document], snapshot) + log = {"schemaVersion": "cultnet.shard_log_response.v0", "messageId": "l", "shardId": "s", "shardEpoch": 1, + "resyncRequired": False, + "entries": [{"sequence": 0, "changeKind": "updated", + "put": {"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p", + "document": raw(schema_id, head + b"\xc0"), "shardId": "s", "shardEpoch": 1}}]} + with self.assertRaises(ValueError, msg=schema_id) as caught: + apply_shard_log_response(cache, [document], log) + self.assertIn("1 bytes after its first MessagePack value", str(caught.exception)) + self.assertIsNone(cache.get(document, "k")) + # A tail on a payload that names no registered schema is still an unknown schema, which a shard log skips. + stranger = msgpack.packb(["tests.stranger.v1", "x"], use_bin_type=True) + b"\xc0" with self.assertRaises(KeyError): - resolve_document_for_raw_record(documents, "unknown", raw("unknown", head + b"\xc0")) - # A registered schema id routes, and the tail is refused by count when the record is applied. - cache = CultCache() - cache.register_document_type(document) - with self.assertRaises(ValueError) as caught: - apply_raw_document_record(cache, documents, raw(known, head + b"\xc0")) - self.assertIn("1 bytes after its first MessagePack value", str(caught.exception)) - self.assertIsNone(cache.get(document, "k")) + resolve_document_for_raw_record(documents, "unknown", raw("unknown", stranger)) def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None: document = define_database_entry_type(