diff --git a/contracts/cultnet/interop/carrier-vectors.cs-written.json b/contracts/cultnet/interop/carrier-vectors.cs-written.json new file mode 100644 index 00000000..394724cf --- /dev/null +++ b/contracts/cultnet/interop/carrier-vectors.cs-written.json @@ -0,0 +1,346 @@ +{ + "vectors": [ + { + "name": "snapshot_request_v1_defaults", + "schemaVersion": "cultnet.snapshot_request.v1", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9uu2N1bHRuZXQuc25hcHNob3RfcmVxdWVzdC52MaltZXNzYWdlSWShbalzZWxlY3Rpb26Jp3NjaGVtYXPApGtleXPApmZpZWxkc8ClY2l0ZXPApWNpdGVkwKpwcm9qZWN0aW9upmhlYWRlcqpkZXNjZW5kaW5nwqVsaW1pdMCmY3Vyc29ywKdzaGFyZElkwKpzaGFyZEVwb2NowA==", + "expected": { + "schemaVersion": "cultnet.snapshot_request.v1", + "messageId": "m", + "selection": { + "schemas": null, + "keys": null, + "fields": null, + "cites": null, + "cited": null, + "projection": "header", + "descending": false, + "limit": null, + "cursor": null + }, + "shardId": null, + "shardEpoch": null + } + }, + { + "name": "snapshot_request_v1_full_selection_and_shard", + "schemaVersion": "cultnet.snapshot_request.v1", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9uu2N1bHRuZXQuc25hcHNob3RfcmVxdWVzdC52MaltZXNzYWdlSWSmbS1mdWxsqXNlbGVjdGlvbomnc2NoZW1hc5KpbGVhZl9hLnY52UdzaGEyNTY6ZmYzNWYzZDlkNmRlM2YxMDY2YjkzNjllNzNmZTZjNzY0OGU4ODM2OGQxOGVmYzIyNzVjOTk0NzMwY2Q4NGNmZqRrZXlzkqRhLWVxqPCfmIAtcm93pmZpZWxkc5KEpWluZGV4pGtpbmSib3CmYW55X29mpnZhbHVlc5Kmd2VhcG9upHRvb2ymbnVtYmVywISlaW5kZXikbWFzc6JvcKJnZaZ2YWx1ZXPApm51bWJlcqstMjIzMzc1OS4yNaVjaXRlc4KmdGFyZ2V0gqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmapcmVjb3JkS2V5pGEtZXGkcm9sZaZEZXNpZ26lY2l0ZWSCpHJvbGWmRGVzaWdupmV4aXN0c8KqcHJvamVjdGlvbqhkb2N1bWVudKpkZXNjZW5kaW5nw6VsaW1pdMzIpmN1cnNvcq1vcGFxdWUtY3Vyc29yp3NoYXJkSWSnc2hhcmQtMapzaGFyZEVwb2NoBw==", + "expected": { + "schemaVersion": "cultnet.snapshot_request.v1", + "messageId": "m-full", + "selection": { + "schemas": [ + "leaf_a.v9", + "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff" + ], + "keys": [ + "a-eq", + "\uD83D\uDE00-row" + ], + "fields": [ + { + "index": "kind", + "op": "any_of", + "values": [ + "weapon", + "tool" + ], + "number": null + }, + { + "index": "mass", + "op": "ge", + "values": null, + "number": "-2233759.25" + } + ], + "cites": { + "target": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "role": "Design" + }, + "cited": { + "role": "Design", + "exists": false + }, + "projection": "document", + "descending": true, + "limit": 200, + "cursor": "opaque-cursor" + }, + "shardId": "shard-1", + "shardEpoch": 7 + } + }, + { + "name": "database_subscribe_v1_defaults", + "schemaVersion": "cultnet.database_subscribe.v1", + "messagePackBase64": "iK1zY2hlbWFWZXJzaW9uvWN1bHRuZXQuZGF0YWJhc2Vfc3Vic2NyaWJlLnYxqW1lc3NhZ2VJZKFtrnN1YnNjcmlwdGlvbklko3N1YqlzZWxlY3Rpb26Jp3NjaGVtYXPApGtleXPApmZpZWxkc8ClY2l0ZXPApWNpdGVkwKpwcm9qZWN0aW9upmhlYWRlcqpkZXNjZW5kaW5nwqVsaW1pdMCmY3Vyc29ywK9pbmNsdWRlU25hcHNob3TDsWNvbnN1bWVyUnVudGltZUlkwKdib2R5SWRzwLdzdXBwb3J0ZWRCb2R5VHJhbnNwb3J0c8A=", + "expected": { + "schemaVersion": "cultnet.database_subscribe.v1", + "messageId": "m", + "subscriptionId": "sub", + "selection": { + "schemas": null, + "keys": null, + "fields": null, + "cites": null, + "cited": null, + "projection": "header", + "descending": false, + "limit": null, + "cursor": null + }, + "includeSnapshot": true, + "consumerRuntimeId": null, + "bodyIds": null, + "supportedBodyTransports": null + } + }, + { + "name": "database_subscribe_v1_full", + "schemaVersion": "cultnet.database_subscribe.v1", + "messagePackBase64": "iK1zY2hlbWFWZXJzaW9uvWN1bHRuZXQuZGF0YWJhc2Vfc3Vic2NyaWJlLnYxqW1lc3NhZ2VJZKZtLWZ1bGyuc3Vic2NyaXB0aW9uSWSoc3ViLWZ1bGypc2VsZWN0aW9uiadzY2hlbWFzkqlsZWFmX2EudjnZR3NoYTI1NjpmZjM1ZjNkOWQ2ZGUzZjEwNjZiOTM2OWU3M2ZlNmM3NjQ4ZTg4MzY4ZDE4ZWZjMjI3NWM5OTQ3MzBjZDg0Y2ZmpGtleXOSpGEtZXGo8J\u002BYgC1yb3emZmllbGRzkoSlaW5kZXika2luZKJvcKZhbnlfb2amdmFsdWVzkqZ3ZWFwb26kdG9vbKZudW1iZXLAhKVpbmRleKRtYXNzom9womdlpnZhbHVlc8CmbnVtYmVyqy0yMjMzNzU5LjI1pWNpdGVzgqZ0YXJnZXSCqHNjaGVtYUlk2UdzaGEyNTY6ZmYzNWYzZDlkNmRlM2YxMDY2YjkzNjllNzNmZTZjNzY0OGU4ODM2OGQxOGVmYzIyNzVjOTk0NzMwY2Q4NGNmZqlyZWNvcmRLZXmkYS1lcaRyb2xlpkRlc2lnbqVjaXRlZIKkcm9sZaZEZXNpZ26mZXhpc3Rzwqpwcm9qZWN0aW9uqGRvY3VtZW50qmRlc2NlbmRpbmfDpWxpbWl0zMimY3Vyc29yrW9wYXF1ZS1jdXJzb3KvaW5jbHVkZVNuYXBzaG90wrFjb25zdW1lclJ1bnRpbWVJZKhjb25zdW1lcqdib2R5SWRzkaZib2R5LWG3c3VwcG9ydGVkQm9keVRyYW5zcG9ydHOSpHJ1ZHCkcXVpYw==", + "expected": { + "schemaVersion": "cultnet.database_subscribe.v1", + "messageId": "m-full", + "subscriptionId": "sub-full", + "selection": { + "schemas": [ + "leaf_a.v9", + "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff" + ], + "keys": [ + "a-eq", + "\uD83D\uDE00-row" + ], + "fields": [ + { + "index": "kind", + "op": "any_of", + "values": [ + "weapon", + "tool" + ], + "number": null + }, + { + "index": "mass", + "op": "ge", + "values": null, + "number": "-2233759.25" + } + ], + "cites": { + "target": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "role": "Design" + }, + "cited": { + "role": "Design", + "exists": false + }, + "projection": "document", + "descending": true, + "limit": 200, + "cursor": "opaque-cursor" + }, + "includeSnapshot": false, + "consumerRuntimeId": "consumer", + "bodyIds": [ + "body-a" + ], + "supportedBodyTransports": [ + "rudp", + "quic" + ] + } + }, + { + "name": "snapshot_response_raw_v1_headers", + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messagePackBase64": "i61zY2hlbWFWZXJzaW9u2SBjdWx0bmV0LnNuYXBzaG90X3Jlc3BvbnNlX3Jhdy52MaltZXNzYWdlSWShbadtYXRjaGVkAqRhc09mBaRuZXh0q25leHQtY3Vyc29yp2hlYWRlcnOSiqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmaqc2NoZW1hTmFtZaZsZWFmX2Gtc2NoZW1hVmVyc2lvbqExsXNjaGVtYUNvbnRlbnRIYXNoo2FiY6lyZWNvcmRLZXmkYS1lcahzdG9yZWRBdLgxOTcwLTAxLTAxVDAwOjAwOjAwLjAwMFqvc291cmNlUnVudGltZUlkonJ0rXNvdXJjZUFnZW50SWSlYWdlbnSqc291cmNlUm9sZaRyb2xlpHRhZ3PAiqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmaqc2NoZW1hTmFtZaZsZWFmX2Gtc2NoZW1hVmVyc2lvbqExsXNjaGVtYUNvbnRlbnRIYXNoo2FiY6lyZWNvcmRLZXmkYS1sb6hzdG9yZWRBdLgxOTcwLTAxLTAxVDAwOjAwOjAwLjAwMFqvc291cmNlUnVudGltZUlkonJ0rXNvdXJjZUFnZW50SWSlYWdlbnSqc291cmNlUm9sZaRyb2xlpHRhZ3OSpWFscGhhpGJldGGpZG9jdW1lbnRzwKVlZGdlc8Cnc2hhcmRJZKdzaGFyZC0xqnNoYXJkRXBvY2gDsHNoYXJkTG9nU2VxdWVuY2Up", + "expected": { + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messageId": "m", + "matched": 2, + "asOf": 5, + "next": "next-cursor", + "headers": [ + { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "schemaName": "leaf_a", + "schemaVersion": "1", + "schemaContentHash": "abc", + "recordKey": "a-eq", + "storedAt": "1970-01-01T00:00:00.000Z", + "sourceRuntimeId": "rt", + "sourceAgentId": "agent", + "sourceRole": "role", + "tags": null + }, + { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "schemaName": "leaf_a", + "schemaVersion": "1", + "schemaContentHash": "abc", + "recordKey": "a-lo", + "storedAt": "1970-01-01T00:00:00.000Z", + "sourceRuntimeId": "rt", + "sourceAgentId": "agent", + "sourceRole": "role", + "tags": [ + "alpha", + "beta" + ] + } + ], + "documents": null, + "edges": null, + "shardId": "shard-1", + "shardEpoch": 3, + "shardLogSequence": 41 + } + }, + { + "name": "snapshot_response_raw_v1_documents_and_edges", + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messagePackBase64": "i61zY2hlbWFWZXJzaW9u2SBjdWx0bmV0LnNuYXBzaG90X3Jlc3BvbnNlX3Jhdy52MaltZXNzYWdlSWShbadtYXRjaGVkAaRhc09mBaRuZXh0wKdoZWFkZXJzwKlkb2N1bWVudHORjKhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmaqc2NoZW1hTmFtZcCtc2NoZW1hVmVyc2lvbsCxc2NoZW1hQ29udGVudEhhc2jAqXJlY29yZEtleaRhLWVxqHN0b3JlZEF0uDE5NzAtMDEtMDFUMDA6MDA6MDAuMDAwWq9wYXlsb2FkRW5jb2RpbmerbWVzc2FnZXBhY2uncGF5bG9hZMQEgaFrAa9zb3VyY2VSdW50aW1lSWTArXNvdXJjZUFnZW50SWTAqnNvdXJjZVJvbGXApHRhZ3PApWVkZ2VzkoWkZnJvbYKoc2NoZW1hSWTZR3NoYTI1Njo5ZDBiYzlhZWJjMzYxOWIwN2FhYmY1NjRmOTg2NjdhZmRlNDE0NDRjMWI4NjY2NjFkNzc3Njg3ZDQ0MzYwOGRjqXJlY29yZEtleadjaXRlci0xpHJvbGWmRGVzaWduonRvgqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmapcmVjb3JkS2V5pGEtZXGvcGF5bG9hZEVuY29kaW5nwKdwYXlsb2FkwIWkZnJvbYKoc2NoZW1hSWTZR3NoYTI1Njo5ZDBiYzlhZWJjMzYxOWIwN2FhYmY1NjRmOTg2NjdhZmRlNDE0NDRjMWI4NjY2NjFkNzc3Njg3ZDQ0MzYwOGRjqXJlY29yZEtleadjaXRlci0xpHJvbGWmRGVzaWduonRvgqhzY2hlbWFJZNlHc2hhMjU2OmZmMzVmM2Q5ZDZkZTNmMTA2NmI5MzY5ZTczZmU2Yzc2NDhlODgzNjhkMThlZmMyMjc1Yzk5NDczMGNkODRjZmapcmVjb3JkS2V5pGEtZXGvcGF5bG9hZEVuY29kaW5nq21lc3NhZ2VwYWNrp3BheWxvYWTEAwECA6dzaGFyZElkwKpzaGFyZEVwb2NowLBzaGFyZExvZ1NlcXVlbmNlwA==", + "expected": { + "schemaVersion": "cultnet.snapshot_response_raw.v1", + "messageId": "m", + "matched": 1, + "asOf": 5, + "next": null, + "headers": null, + "documents": [ + { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "schemaName": null, + "schemaVersion": null, + "schemaContentHash": null, + "recordKey": "a-eq", + "storedAt": "1970-01-01T00:00:00.000Z", + "payloadEncoding": "messagepack", + "payload": "gaFrAQ==", + "sourceRuntimeId": null, + "sourceAgentId": null, + "sourceRole": null, + "tags": null + } + ], + "edges": [ + { + "from": { + "schemaId": "sha256:9d0bc9aebc3619b07aabf564f98667afde41444c1b866661d777687d443608dc", + "recordKey": "citer-1" + }, + "role": "Design", + "to": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "payloadEncoding": null, + "payload": null + }, + { + "from": { + "schemaId": "sha256:9d0bc9aebc3619b07aabf564f98667afde41444c1b866661d777687d443608dc", + "recordKey": "citer-1" + }, + "role": "Design", + "to": { + "schemaId": "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff", + "recordKey": "a-eq" + }, + "payloadEncoding": "messagepack", + "payload": "AQID" + } + ], + "shardId": null, + "shardEpoch": null, + "shardLogSequence": null + } + }, + { + "name": "error_plain", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3KkYm9vbatyb3V0aW5nSGludMCkY29kZcCnZGV0YWlsc8A=", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "boom", + "routingHint": null, + "code": null, + "details": null + } + }, + { + "name": "error_selection_invalid", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3LZOnNlbGVjdGlvbl9pbnZhbGlkOiBmaWVsZHNbMF0uaW5kZXggImtpbmQiIGlzIG5vdCBkZWNsYXJlZC6rcm91dGluZ0hpbnTApGNvZGWxc2VsZWN0aW9uX2ludmFsaWSnZGV0YWlsc4SlZmllbGSvZmllbGRzWzBdLmluZGV4pXZhbHVlpGtpbmSkYXNPZsCnY3VycmVudMA=", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "selection_invalid: fields[0].index \u0022kind\u0022 is not declared.", + "routingHint": null, + "code": "selection_invalid", + "details": { + "field": "fields[0].index", + "value": "kind", + "asOf": null, + "current": null + } + } + }, + { + "name": "error_cursor_stale", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3KzY3Vyc29yX3N0YWxlOiBzdGFsZatyb3V0aW5nSGludMCkY29kZaxjdXJzb3Jfc3RhbGWnZGV0YWlsc4SlZmllbGTApXZhbHVlwKRhc09mA6djdXJyZW50CQ==", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_stale: stale", + "routingHint": null, + "code": "cursor_stale", + "details": { + "field": null, + "value": null, + "asOf": 3, + "current": 9 + } + } + }, + { + "name": "error_cursor_invalid_no_details", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3KzY3Vyc29yX2ludmFsaWQ6IGJhZKtyb3V0aW5nSGludMCkY29kZa5jdXJzb3JfaW52YWxpZKdkZXRhaWxzwA==", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_invalid: bad", + "routingHint": null, + "code": "cursor_invalid", + "details": null + } + }, + { + "name": "error_write_not_authorized", + "schemaVersion": "cultnet.error.v0", + "messagePackBase64": "ha1zY2hlbWFWZXJzaW9usGN1bHRuZXQuZXJyb3IudjClZXJyb3K0d3JpdGVfbm90X2F1dGhvcml6ZWSrcm91dGluZ0hpbnTApGNvZGW0d3JpdGVfbm90X2F1dGhvcml6ZWSnZGV0YWlsc4SlZmllbGSpcmVjb3JkS2V5pXZhbHVloWukYXNPZsCnY3VycmVudMA=", + "expected": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized", + "routingHint": null, + "code": "write_not_authorized", + "details": { + "field": "recordKey", + "value": "k", + "asOf": null, + "current": null + } + } + } + ] +} \ No newline at end of file diff --git a/packages/cultmesh-py/README.md b/packages/cultmesh-py/README.md index 63799e52..4a048dcc 100644 --- a/packages/cultmesh-py/README.md +++ b/packages/cultmesh-py/README.md @@ -228,6 +228,17 @@ prediction = session.predict(note_doc, "input:python:move", {"body": "predicted unsubscribe() ``` +## Authority route verification + +`cultmesh_py.authority` ports the C# reference +(`src/GameCult.Mesh/CultMeshAuthorityProof.cs`): `AuthorityTrustPolicy.validate` +judges an Odin-signed ECDSA P-256 route (channel protection, trusted root, +validity window, IEEE P1363 signature over the canonical bytes) and +`verify_session_proof` checks the provider nonce proof. Refusals raise +`AuthorityRouteRefusal` with the C# message text. It needs `cryptography`. +`tests/test_authority_route_vectors.py` judges +`contracts/cultmesh/authority-route-vectors*.json`. + ## Verify `cultmesh_py.verify` checks the public exports and `py.typed` markers of all diff --git a/packages/cultmesh-py/pyproject.toml b/packages/cultmesh-py/pyproject.toml index a1df5ee3..5f9bda4d 100644 --- a/packages/cultmesh-py/pyproject.toml +++ b/packages/cultmesh-py/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0"] +dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0", "cryptography>=41"] [project.scripts] cultmesh-py-daemon = "cultmesh_py.daemon:main" cultmesh-py-verify = "cultmesh_py.verify:main" diff --git a/packages/cultmesh-py/src/cultmesh_py/authority.py b/packages/cultmesh-py/src/cultmesh_py/authority.py new file mode 100644 index 00000000..3c6665ea --- /dev/null +++ b/packages/cultmesh-py/src/cultmesh_py/authority.py @@ -0,0 +1,338 @@ +"""CultMesh authority route verification. + +Port of `src/GameCult.Mesh/CultMeshAuthorityProof.cs` (the reference) and of the +route constructors in `CultMeshVerse.cs`. Rule for rule, in the same order, the +same input yields the same refusal text as `CultMeshAuthorityTrustPolicy.Verify`: +unsigned, channel protection, root lookup, validity window, signature. + +Shared vectors: `contracts/cultmesh/authority-route-vectors.json` (C# written). +""" + +from __future__ import annotations + +import base64 +import binascii +import re +import struct +from dataclasses import dataclass +from typing import Iterable, Mapping, Sequence + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature + +ROUTE_DOMAIN = "gamecult.cultmesh.route-certificate.v1" +SESSION_DOMAIN = "gamecult.cultmesh.session-proof.v1" + +AUTHENTICATED_REMOTE = "authenticated-remote" +LOCAL_DEVELOPMENT = "local-development" + +# .NET `char.IsWhiteSpace`: Unicode Zs, Zl, Zp, U+0009-U+000D and U+0085. +# `str.strip()` is a different set (it also strips U+001C-U+001F). +_CSHARP_WHITESPACE = ( + "\u0009\u000a\u000b\u000c\u000d \u0085  " + "           " + "

  　" +) + + +def trim_csharp(value: str) -> str: + return value.strip(_CSHARP_WHITESPACE) + + +def is_blank_csharp(value: str | None) -> bool: + return value is None or trim_csharp(value) == "" + + +def _require(value: str | None, name: str) -> str: + if is_blank_csharp(value): + raise ValueError(f"Value must be non-empty. (Parameter '{name}')") + return trim_csharp(value) # type: ignore[arg-type] + + +class AuthorityRouteRefusal(Exception): + """A route or proof the consumer's trust policy refuses. + + Mirrors `CultMeshSessionException` carrying a `CultMeshSessionFailure` with + reason `Authentication`: `reason` is always "authentication". + """ + + reason = "authentication" + + def __init__(self, message: str, endpoint: str = "") -> None: + super().__init__(message) + self.message = message + self.endpoint = endpoint + + +def _b64_strict(value: str) -> bytes: + # `Convert.FromBase64String`: standard alphabet, padded, whitespace ignored. + stripped = re.sub(r"[ \t\r\n]", "", value) + return base64.b64decode(stripped, validate=True) + + +@dataclass(frozen=True) +class P256PublicKey: + key_id: str + x: str + y: str + + def __post_init__(self) -> None: + object.__setattr__(self, "key_id", _require(self.key_id, "keyId")) + for name in ("x", "y"): + value = _require(getattr(self, name), name) + try: + _b64_strict(value) + except (binascii.Error, ValueError) as error: + raise ValueError(f"Coordinate must be base64. (Parameter '{name}')") from error + object.__setattr__(self, name, value) + + def verify(self, payload: bytes, signature_base64: str) -> bool: + """IEEE P1363 (r || s) ECDSA-SHA256 verification; False on any malformation.""" + try: + x, y = _b64_strict(self.x), _b64_strict(self.y) + signature = _b64_strict(signature_base64) + if len(x) != 32 or len(y) != 32 or len(signature) != 64: + return False + key = ec.EllipticCurvePublicNumbers( + int.from_bytes(x, "big"), int.from_bytes(y, "big"), ec.SECP256R1() + ).public_key() + der = encode_dss_signature( + int.from_bytes(signature[:32], "big"), int.from_bytes(signature[32:], "big") + ) + key.verify(der, payload, ec.ECDSA(hashes.SHA256())) + return True + except (InvalidSignature, ValueError, binascii.Error): + return False + + +@dataclass(frozen=True) +class RouteCertificate: + provider_key: P256PublicKey + odin_key_id: str + issued_at_unix_milliseconds: int + expires_at_unix_milliseconds: int + signature: str = "" + + def __post_init__(self) -> None: + if is_blank_csharp(self.odin_key_id): + raise ValueError("Odin key identity is required. (Parameter 'odinKeyId')") + object.__setattr__(self, "odin_key_id", trim_csharp(self.odin_key_id)) + if self.issued_at_unix_milliseconds < 0: + raise ValueError("issuedAtUnixMilliseconds must not be negative.") + if self.expires_at_unix_milliseconds <= self.issued_at_unix_milliseconds: + raise ValueError("expiresAtUnixMilliseconds must be after issuedAtUnixMilliseconds.") + object.__setattr__(self, "signature", trim_csharp(self.signature or "")) + + +@dataclass(frozen=True) +class AuthorityRoute: + """`CultMeshAuthorityRoute`: construction cleans every field as the C# does.""" + + authority_runtime_id: str + endpoint: str + protocol_ids: Sequence[str] = () + priority: int = 0 + generation: str | None = None + certificate: RouteCertificate | None = None + + def __post_init__(self) -> None: + runtime_id = _require(self.authority_runtime_id, "authorityRuntimeId") + endpoint = _require(self.endpoint, "endpoint") + if self.priority < 0: + raise ValueError("priority must not be negative.") + object.__setattr__(self, "authority_runtime_id", runtime_id) + object.__setattr__(self, "endpoint", endpoint) + object.__setattr__( + self, + "protocol_ids", + tuple( + sorted( + {trim_csharp(v) for v in self.protocol_ids if not is_blank_csharp(v)}, + key=lambda v: v.encode("utf-16-be"), # StringComparer.Ordinal is UTF-16 code-unit order + ) + ), + ) + object.__setattr__( + self, + "generation", + f"{runtime_id}@{endpoint}" if is_blank_csharp(self.generation) else trim_csharp(self.generation), # type: ignore[arg-type] + ) + + +@dataclass(frozen=True) +class SessionOpen: + """The fields of `CultMeshSessionOpenMessage` the session proof covers.""" + + client_nonce: str | None = None + message_id: str | None = None + source_runtime_id: str | None = None + verse_id: str | None = None + authority_runtime_id: str | None = None + protocol_id: str | None = None + route_generation: str | None = None + + +@dataclass(frozen=True) +class SessionAccepted: + client_nonce: str | None + provider_key_id: str | None + provider_signature: str | None + + +@dataclass(frozen=True) +class VerifiedRoute: + route: AuthorityRoute + provider_key: P256PublicKey | None + is_local_development: bool + + +def canonical_fields(*values: str | None) -> bytes: + out = bytearray() + for value in values: + encoded = (value or "").encode("utf-8") + out += struct.pack(">I", len(encoded)) + encoded + return bytes(out) + + +def canonical_route(verse_id: str, route: AuthorityRoute) -> bytes: + certificate = route.certificate + if certificate is None: + raise ValueError("Route certificate is required.") + return canonical_fields( + ROUTE_DOMAIN, + verse_id, + route.authority_runtime_id, + route.endpoint, + "\u001f".join(route.protocol_ids), # already ordinally sorted; Python sorts by code point + str(route.priority), + route.generation, + certificate.provider_key.key_id, + certificate.provider_key.x, + certificate.provider_key.y, + certificate.odin_key_id, + str(certificate.issued_at_unix_milliseconds), + str(certificate.expires_at_unix_milliseconds), + ) + + +def canonical_session(request: SessionOpen, endpoint: str) -> bytes: + return canonical_fields( + SESSION_DOMAIN, + request.client_nonce, + request.message_id, + request.source_runtime_id, + request.verse_id, + request.authority_runtime_id, + request.protocol_id, + endpoint, + request.route_generation, + ) + + +# `System.Uri` refuses spellings WHATWG parsers repair; a lenient parse could call +# a refused endpoint loopback or protected. Same shape rule as the TypeScript port. +_AUTHORITY = re.compile(r"^[A-Za-z][A-Za-z0-9+\-.]*://([^/?#]+)") + + +def _split_endpoint(value: str) -> tuple[str, str] | None: + """Return (scheme, lowercase host) or None where System.Uri would refuse.""" + raw = re.sub(r"^[\x00-\x20]+|[\x00-\x20]+$", "", value) + if "\\" in raw: + return None + shape = _AUTHORITY.match(raw) + if not shape: + return None + authority = shape.group(1) + after_user = authority[authority.rfind("@") + 1 :] + if after_user.startswith("["): + host = after_user[: after_user.find("]") + 1] + else: + host = after_user.split(":")[0] + if not host or re.search(r"[^!-~]", host) or host.endswith("."): + return None + return raw.split("://", 1)[0].lower(), host.strip("[]").lower() + + +def is_loopback_endpoint(endpoint: str) -> bool: + parts = _split_endpoint(endpoint) + if parts is None: + return False + host = parts[1] + return host in ("localhost", "loopback", "::1", "::ffff:127.0.0.1") or bool( + re.fullmatch(r"127\.\d{1,3}\.\d{1,3}\.\d{1,3}", host) + ) + + +def is_protected_endpoint(endpoint: str) -> bool: + parts = _split_endpoint(endpoint) + return parts is not None and (parts[0] in ("wss", "https") or "quic" in parts[0]) + + +class AuthorityTrustPolicy: + """Consumer-owned policy. Fails closed unless a route is valid under it.""" + + def __init__(self, mode: str, odin_roots: Iterable[P256PublicKey] = ()) -> None: + if mode not in (AUTHENTICATED_REMOTE, LOCAL_DEVELOPMENT): + raise ValueError(f"Unknown trust mode '{mode}'.") + self.mode = mode + self._roots: dict[str, P256PublicKey] = {} + for root in odin_roots: + if root.key_id in self._roots: + raise ValueError(f"Odin root key id '{root.key_id}' is listed more than once in the trust policy.") + self._roots[root.key_id] = root + + def validate(self, verse_id: str, route: AuthorityRoute, now_unix_milliseconds: int) -> None: + self.verify(verse_id, route, now_unix_milliseconds) + + def verify(self, verse_id: str, route: AuthorityRoute, now_unix_milliseconds: int) -> VerifiedRoute: + local = self.mode == LOCAL_DEVELOPMENT and is_loopback_endpoint(route.endpoint) + + def refuse(message: str) -> AuthorityRouteRefusal: + return AuthorityRouteRefusal(message, route.endpoint) + + certificate = route.certificate + if certificate is None or is_blank_csharp(certificate.signature): + if local: + return VerifiedRoute(route, None, True) + raise refuse("Remote CultMesh routes require an Odin-signed authority certificate.") + if not is_protected_endpoint(route.endpoint) and not local: + raise refuse("Authenticated remote CultMesh routes require TLS or QUIC channel protection.") + root = self._roots.get(certificate.odin_key_id) + if root is None: + raise refuse(f"Odin key '{certificate.odin_key_id}' is not trusted by this consumer.") + if not (certificate.issued_at_unix_milliseconds <= now_unix_milliseconds < certificate.expires_at_unix_milliseconds): + raise refuse("The Odin route certificate is not currently valid.") + try: + signature = _b64_strict(certificate.signature) + except (binascii.Error, ValueError): + raise refuse("The Odin route signature is not base64.") from None + if len(signature) != 64: + raise refuse("The Odin route signature is not IEEE P1363 P-256.") + if not root.verify(canonical_route(verse_id, route), certificate.signature): + raise refuse("The Odin route certificate signature is invalid.") + return VerifiedRoute(route, certificate.provider_key, False) + + +def verify_session_proof( + request: SessionOpen, + response: SessionAccepted, + verse_id: str, + route: AuthorityRoute, + trust: AuthorityTrustPolicy, + now_unix_milliseconds: int, +) -> bool: + """Verifies the Odin route certificate (raising) and the provider nonce proof.""" + verified = trust.verify(verse_id, route, now_unix_milliseconds) + if verified.is_local_development: + return True + key = verified.provider_key + if ( + key is None + or response.provider_key_id != key.key_id + or response.client_nonce != request.client_nonce + or is_blank_csharp(response.provider_signature) + ): + return False + return key.verify(canonical_session(request, route.endpoint), response.provider_signature) # type: ignore[arg-type] diff --git a/packages/cultmesh-py/tests/test_authority_route_vectors.py b/packages/cultmesh-py/tests/test_authority_route_vectors.py new file mode 100644 index 00000000..3f116e2d --- /dev/null +++ b/packages/cultmesh-py/tests/test_authority_route_vectors.py @@ -0,0 +1,217 @@ +"""Judges `contracts/cultmesh/authority-route-vectors*.json` in Python. + +`authority-route-vectors.json` is written by the C# reference (signatures are +captured C# bytes); the `.ts-signed.json` file is signed by TypeScript over awkward +inputs. Each negative case fails when the matching check is removed from +`cultmesh_py.authority`. +""" + +from __future__ import annotations + +import base64 +import json +import unittest +from dataclasses import replace +from pathlib import Path + +from cryptography.hazmat.primitives.asymmetric import ec + +from cultmesh_py.authority import ( + AUTHENTICATED_REMOTE, + LOCAL_DEVELOPMENT, + AuthorityRoute, + AuthorityRouteRefusal, + AuthorityTrustPolicy, + P256PublicKey, + RouteCertificate, + SessionAccepted, + SessionOpen, + canonical_route, + verify_session_proof, +) + +CONTRACTS = Path(__file__).resolve().parents[3] / "contracts" / "cultmesh" + + +def _key(element: dict) -> P256PublicKey: + return P256PublicKey(element["keyId"], element["x"], element["y"]) + + +class Vector: + def __init__(self, name: str) -> None: + doc = json.loads((CONTRACTS / name).read_text(encoding="utf-8")) + self.now = doc["nowUnixMilliseconds"] + self.root = _key(doc["odinRoot"]) + r = doc["route"] + self.verse_id = r["verseId"] + c = r["certificate"] + self.certificate = RouteCertificate( + _key(c["providerKey"]), c["odinKeyId"], c["issuedAtUnixMilliseconds"], + c["expiresAtUnixMilliseconds"], c["signature"], + ) + self.route = AuthorityRoute( + r["authorityRuntimeId"], r["endpoint"], r["protocolIds"], r["priority"], + r["generation"], self.certificate, + ) + p = doc["sessionProof"] + q = p["request"] + self.request = SessionOpen( + q["clientNonce"], q["messageId"], q["sourceRuntimeId"], q["verseId"], + q["authorityRuntimeId"], q["protocolId"], q["routeGeneration"], + ) + self.accepted = SessionAccepted(q["clientNonce"], p["providerKeyId"], p["providerSignature"]) + self.trust = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, [self.root]) + + def with_certificate(self, **changes) -> AuthorityRoute: + return replace(self.route, certificate=replace(self.certificate, **changes)) + + +def refusal(test: unittest.TestCase, fn) -> str: + with test.assertRaises(AuthorityRouteRefusal) as raised: + fn() + test.assertEqual("authentication", raised.exception.reason) + return raised.exception.message + + +class AuthorityRouteVectorTests(unittest.TestCase): + def test_cs_written_vector_verifies_route_and_session_proof(self) -> None: + v = Vector("authority-route-vectors.json") + v.trust.validate(v.verse_id, v.route, v.now) + self.assertTrue(verify_session_proof(v.request, v.accepted, v.verse_id, v.route, v.trust, v.now)) + + def test_ts_signed_vector_verifies_and_tamper_is_refused(self) -> None: + v = Vector("authority-route-vectors.ts-signed.json") + v.trust.validate(v.verse_id, v.route, v.now) + self.assertTrue(verify_session_proof(v.request, v.accepted, v.verse_id, v.route, v.trust, v.now)) + tampered = replace(v.route, priority=v.route.priority + 1) + self.assertEqual( + "The Odin route certificate signature is invalid.", + refusal(self, lambda: v.trust.validate(v.verse_id, tampered, v.now)), + ) + + def test_canonical_route_is_length_prefixed_and_sorted(self) -> None: + v = Vector("authority-route-vectors.json") + data = canonical_route(v.verse_id, v.route) + self.assertTrue(data.startswith(b"\x00\x00\x00\x26gamecult.cultmesh.route-certificate.v1\x00\x00\x00\x08aetheria")) + self.assertIn(b"cultmesh.content.v1\x1fcultmesh.documents.v1", data) + + def test_tampered_payload_fields_are_refused(self) -> None: + v = Vector("authority-route-vectors.json") + invalid = "The Odin route certificate signature is invalid." + for tampered, verse in ( + (replace(v.route, priority=8), v.verse_id), + (replace(v.route, endpoint="wss://evil.example/mesh"), v.verse_id), + (replace(v.route, protocol_ids=("cultmesh.documents.v1",)), v.verse_id), + (replace(v.route, generation="other"), v.verse_id), + (v.with_certificate(expires_at_unix_milliseconds=v.certificate.expires_at_unix_milliseconds + 1), v.verse_id), + (v.route, "other-verse"), + ): + self.assertEqual(invalid, refusal(self, lambda: v.trust.validate(verse, tampered, v.now))) + + def test_wrong_root_key_is_refused(self) -> None: + v = Vector("authority-route-vectors.json") + numbers = ec.generate_private_key(ec.SECP256R1()).public_key().public_numbers() + impostor = P256PublicKey( + v.root.key_id, + base64.b64encode(numbers.x.to_bytes(32, "big")).decode(), + base64.b64encode(numbers.y.to_bytes(32, "big")).decode(), + ) + trust = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, [impostor]) + self.assertEqual( + "The Odin route certificate signature is invalid.", + refusal(self, lambda: trust.validate(v.verse_id, v.route, v.now)), + ) + + def test_untrusted_odin_key_id_is_refused(self) -> None: + v = Vector("authority-route-vectors.json") + trust = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, [replace(v.root, key_id="someone-else")]) + self.assertEqual( + "Odin key 'odin-vector' is not trusted by this consumer.", + refusal(self, lambda: trust.validate(v.verse_id, v.route, v.now)), + ) + + def test_expired_and_not_yet_valid_are_refused(self) -> None: + v = Vector("authority-route-vectors.json") + window = "The Odin route certificate is not currently valid." + expires = v.certificate.expires_at_unix_milliseconds + issued = v.certificate.issued_at_unix_milliseconds + v.trust.validate(v.verse_id, v.route, expires - 1) + v.trust.validate(v.verse_id, v.route, issued) + self.assertEqual(window, refusal(self, lambda: v.trust.validate(v.verse_id, v.route, expires))) + self.assertEqual(window, refusal(self, lambda: v.trust.validate(v.verse_id, v.route, issued - 1))) + + def test_unsigned_and_unprotected_routes_are_refused(self) -> None: + v = Vector("authority-route-vectors.json") + unsigned = replace(v.route, certificate=None) + self.assertEqual( + "Remote CultMesh routes require an Odin-signed authority certificate.", + refusal(self, lambda: v.trust.validate(v.verse_id, unsigned, v.now)), + ) + blank = v.with_certificate(signature=" ") + self.assertEqual( + "Remote CultMesh routes require an Odin-signed authority certificate.", + refusal(self, lambda: v.trust.validate(v.verse_id, blank, v.now)), + ) + plain = replace(v.route, endpoint="ws://provider.example/mesh") + self.assertEqual( + "Authenticated remote CultMesh routes require TLS or QUIC channel protection.", + refusal(self, lambda: v.trust.validate(v.verse_id, plain, v.now)), + ) + + def test_local_development_admits_only_unsigned_loopback(self) -> None: + v = Vector("authority-route-vectors.json") + dev = AuthorityTrustPolicy(LOCAL_DEVELOPMENT, []) + loop = AuthorityRoute("a", "ws://127.0.0.1:9/mesh") + dev.validate(v.verse_id, loop, v.now) + self.assertTrue(verify_session_proof(v.request, v.accepted, v.verse_id, loop, dev, v.now)) + for endpoint in ("ws://provider.example/mesh", "ws://127.0.0.1./mesh", "ws:127.0.0.1"): + refusal(self, lambda e=endpoint: dev.validate(v.verse_id, AuthorityRoute("a", e), v.now)) + authenticated = AuthorityTrustPolicy(AUTHENTICATED_REMOTE, []) + refusal(self, lambda: authenticated.validate(v.verse_id, loop, v.now)) + + def test_malformed_signatures_get_their_own_refusals(self) -> None: + v = Vector("authority-route-vectors.json") + not_base64 = v.with_certificate(signature="!!not base64!!") + self.assertEqual( + "The Odin route signature is not base64.", + refusal(self, lambda: v.trust.validate(v.verse_id, not_base64, v.now)), + ) + short = v.with_certificate(signature=base64.b64encode(b"x" * 63).decode()) + self.assertEqual( + "The Odin route signature is not IEEE P1363 P-256.", + refusal(self, lambda: v.trust.validate(v.verse_id, short, v.now)), + ) + + def test_high_s_signature_verifies_as_it_does_in_dotnet(self) -> None: + v = Vector("authority-route-vectors.json") + order = 0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551 + raw = base64.b64decode(v.certificate.signature) + s = int.from_bytes(raw[32:], "big") + flipped = raw[:32] + (order - s).to_bytes(32, "big") + high = v.with_certificate(signature=base64.b64encode(flipped).decode()) + v.trust.validate(v.verse_id, high, v.now) + + def test_session_proof_rejects_replay_wrong_key_and_forgery(self) -> None: + v = Vector("authority-route-vectors.json") + replayed = replace(v.request, client_nonce="replayed") + accepted = replace(v.accepted, client_nonce="replayed") + self.assertFalse(verify_session_proof(replayed, accepted, v.verse_id, v.route, v.trust, v.now)) + self.assertFalse(verify_session_proof( + v.request, replace(v.accepted, provider_key_id="other"), v.verse_id, v.route, v.trust, v.now)) + self.assertFalse(verify_session_proof( + v.request, replace(v.accepted, client_nonce="other"), v.verse_id, v.route, v.trust, v.now)) + self.assertFalse(verify_session_proof( + v.request, replace(v.accepted, provider_signature=""), v.verse_id, v.route, v.trust, v.now)) + raw = bytearray(base64.b64decode(v.accepted.provider_signature)) + raw[0] ^= 1 + forged = replace(v.accepted, provider_signature=base64.b64encode(bytes(raw)).decode()) + self.assertFalse(verify_session_proof(v.request, forged, v.verse_id, v.route, v.trust, v.now)) + moved = replace(v.request, protocol_id="cultmesh.content.v1") + self.assertFalse(verify_session_proof(moved, v.accepted, v.verse_id, v.route, v.trust, v.now)) + # An expired route fails closed even with a good proof. + refusal(self, lambda: verify_session_proof( + v.request, v.accepted, v.verse_id, v.route, v.trust, v.certificate.expires_at_unix_milliseconds)) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultmesh-rs/Cargo.lock b/packages/cultmesh-rs/Cargo.lock index d056b52e..de7c506e 100644 --- a/packages/cultmesh-rs/Cargo.lock +++ b/packages/cultmesh-rs/Cargo.lock @@ -67,6 +67,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" @@ -161,6 +167,18 @@ dependencies = [ "libc", ] +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -214,11 +232,14 @@ name = "cultmesh-rs" version = "0.2.0" dependencies = [ "anyhow", + "base64", "cultcache-rs", "cultnet-rs", + "p256", "pretty_assertions", "rmp-serde", "serde", + "serde_json", "tempfile", ] @@ -298,10 +319,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -326,6 +362,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "errno" version = "0.3.14" @@ -342,6 +397,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -396,6 +461,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -442,6 +508,17 @@ dependencies = [ "polyval", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "hmac" version = "0.12.1" @@ -555,6 +632,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -602,6 +691,15 @@ dependencies = [ "yansi", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -689,6 +787,16 @@ dependencies = [ "libc", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "rmp" version = "0.8.15" @@ -747,6 +855,20 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "semver" version = "1.0.28" @@ -840,6 +962,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ + "digest", "rand_core 0.6.4", ] diff --git a/packages/cultmesh-rs/Cargo.toml b/packages/cultmesh-rs/Cargo.toml index af75e419..984b74ab 100644 --- a/packages/cultmesh-rs/Cargo.toml +++ b/packages/cultmesh-rs/Cargo.toml @@ -13,9 +13,12 @@ path = "src/lib.rs" anyhow = "1" cultcache-rs = { path = "../cultcache-rs" } cultnet-rs = { path = "../cultnet-rs" } +base64 = "0.22" +p256 = { version = "0.13", default-features = false, features = ["ecdsa", "std"] } serde = { version = "1", features = ["derive"] } [dev-dependencies] pretty_assertions = "1.4.1" rmp-serde = "1" +serde_json = "1" tempfile = "3" diff --git a/packages/cultmesh-rs/src/authority.rs b/packages/cultmesh-rs/src/authority.rs new file mode 100644 index 00000000..f4368884 --- /dev/null +++ b/packages/cultmesh-rs/src/authority.rs @@ -0,0 +1,424 @@ +//! CultMesh authority route verification. +//! +//! Port of `src/GameCult.Mesh/CultMeshAuthorityProof.cs` (the reference) and of +//! the route constructors in `CultMeshVerse.cs`. Rule for rule, in the same +//! order, the same input yields the same refusal text as +//! `CultMeshAuthorityTrustPolicy.Verify`: unsigned, channel protection, root +//! lookup, validity window, signature. +//! +//! Shared vectors: `contracts/cultmesh/authority-route-vectors.json` (C# written). + +use base64::alphabet::STANDARD as STANDARD_ALPHABET; +use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; +use base64::Engine; +use p256::ecdsa::signature::Verifier; +use p256::ecdsa::{Signature, VerifyingKey}; +use std::collections::BTreeMap; +use std::fmt; + +const ROUTE_DOMAIN: &str = "gamecult.cultmesh.route-certificate.v1"; +const SESSION_DOMAIN: &str = "gamecult.cultmesh.session-proof.v1"; + +// `Convert.FromBase64String`: standard alphabet, padded, tolerant of trailing bits. +const DOTNET_BASE64: GeneralPurpose = GeneralPurpose::new( + &STANDARD_ALPHABET, + GeneralPurposeConfig::new() + .with_decode_padding_mode(DecodePaddingMode::RequireCanonical) + .with_decode_allow_trailing_bits(true), +); + +fn base64_decode(value: &str) -> Option> { + let stripped: String = value.chars().filter(|c| !matches!(c, ' ' | '\t' | '\r' | '\n')).collect(); + DOTNET_BASE64.decode(stripped).ok() +} + +/// .NET `char.IsWhiteSpace` is the Unicode White_Space property, as is `char::is_whitespace`. +fn trim(value: &str) -> &str { + value.trim_matches(char::is_whitespace) +} + +fn is_blank(value: &str) -> bool { + trim(value).is_empty() +} + +fn require(value: &str, name: &str) -> Result { + if is_blank(value) { + Err(format!("Value must be non-empty. (Parameter '{name}')")) + } else { + Ok(trim(value).to_owned()) + } +} + +/// A route or proof the consumer's trust policy refuses. Mirrors +/// `CultMeshSessionFailure` with reason `Authentication`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuthorityRouteRefusal { + pub message: String, + pub endpoint: String, +} + +impl AuthorityRouteRefusal { + pub const REASON: &'static str = "authentication"; +} + +impl fmt::Display for AuthorityRouteRefusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for AuthorityRouteRefusal {} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct P256PublicKey { + pub key_id: String, + pub x: String, + pub y: String, +} + +impl P256PublicKey { + pub fn new(key_id: &str, x: &str, y: &str) -> Result { + let key = Self { + key_id: require(key_id, "keyId")?, + x: require(x, "x")?, + y: require(y, "y")?, + }; + for (name, value) in [("x", &key.x), ("y", &key.y)] { + if base64_decode(value).is_none() { + return Err(format!("Coordinate must be base64. (Parameter '{name}')")); + } + } + Ok(key) + } + + /// IEEE P1363 (r || s) ECDSA-SHA256 verification; false on any malformation. + pub fn verify(&self, payload: &[u8], signature_base64: &str) -> bool { + let (Some(x), Some(y), Some(signature)) = ( + base64_decode(&self.x), + base64_decode(&self.y), + base64_decode(signature_base64), + ) else { + return false; + }; + if x.len() != 32 || y.len() != 32 || signature.len() != 64 { + return false; + } + let mut sec1 = Vec::with_capacity(65); + sec1.push(4); + sec1.extend_from_slice(&x); + sec1.extend_from_slice(&y); + let (Ok(key), Ok(signature)) = ( + VerifyingKey::from_sec1_bytes(&sec1), + Signature::from_slice(&signature), + ) else { + return false; + }; + key.verify(payload, &signature).is_ok() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RouteCertificate { + pub provider_key: P256PublicKey, + pub odin_key_id: String, + pub issued_at_unix_milliseconds: i64, + pub expires_at_unix_milliseconds: i64, + pub signature: String, +} + +impl RouteCertificate { + pub fn new( + provider_key: P256PublicKey, + odin_key_id: &str, + issued_at_unix_milliseconds: i64, + expires_at_unix_milliseconds: i64, + signature: &str, + ) -> Result { + if is_blank(odin_key_id) { + return Err("Odin key identity is required. (Parameter 'odinKeyId')".into()); + } + if issued_at_unix_milliseconds < 0 { + return Err("issuedAtUnixMilliseconds must not be negative.".into()); + } + if expires_at_unix_milliseconds <= issued_at_unix_milliseconds { + return Err("expiresAtUnixMilliseconds must be after issuedAtUnixMilliseconds.".into()); + } + Ok(Self { + provider_key, + odin_key_id: trim(odin_key_id).to_owned(), + issued_at_unix_milliseconds, + expires_at_unix_milliseconds, + signature: trim(signature).to_owned(), + }) + } +} + +/// `CultMeshAuthorityRoute`: construction cleans every field as the C# does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AuthorityRoute { + pub authority_runtime_id: String, + pub endpoint: String, + pub protocol_ids: Vec, + pub priority: i32, + pub generation: String, + pub certificate: Option, +} + +impl AuthorityRoute { + pub fn new( + authority_runtime_id: &str, + endpoint: &str, + protocol_ids: &[&str], + priority: i32, + generation: Option<&str>, + certificate: Option, + ) -> Result { + let authority_runtime_id = require(authority_runtime_id, "authorityRuntimeId")?; + let endpoint = require(endpoint, "endpoint")?; + if priority < 0 { + return Err("priority must not be negative.".into()); + } + let mut protocols: Vec = protocol_ids + .iter() + .filter(|value| !is_blank(value)) + .map(|value| trim(value).to_owned()) + .collect(); + // StringComparer.Ordinal is UTF-16 code-unit order. + protocols.sort_by_key(|value| value.encode_utf16().collect::>()); + protocols.dedup(); + let generation = match generation { + Some(value) if !is_blank(value) => trim(value).to_owned(), + _ => format!("{authority_runtime_id}@{endpoint}"), + }; + Ok(Self { + authority_runtime_id, + endpoint, + protocol_ids: protocols, + priority, + generation, + certificate, + }) + } +} + +/// The fields of `CultMeshSessionOpenMessage` the session proof covers. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionOpen { + pub client_nonce: String, + pub message_id: String, + pub source_runtime_id: String, + pub verse_id: String, + pub authority_runtime_id: String, + pub protocol_id: String, + pub route_generation: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SessionAccepted { + pub client_nonce: String, + pub provider_key_id: String, + pub provider_signature: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TrustMode { + AuthenticatedRemote, + LocalDevelopment, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VerifiedRoute { + pub route: AuthorityRoute, + pub provider_key: Option, + pub is_local_development: bool, +} + +pub fn canonical_fields(values: &[&str]) -> Vec { + let mut out = Vec::new(); + for value in values { + out.extend_from_slice(&(value.len() as u32).to_be_bytes()); + out.extend_from_slice(value.as_bytes()); + } + out +} + +pub fn canonical_route(verse_id: &str, route: &AuthorityRoute) -> Result, String> { + let certificate = route.certificate.as_ref().ok_or("Route certificate is required.")?; + let protocols = route.protocol_ids.join("\u{1f}"); + let priority = route.priority.to_string(); + let issued = certificate.issued_at_unix_milliseconds.to_string(); + let expires = certificate.expires_at_unix_milliseconds.to_string(); + Ok(canonical_fields(&[ + ROUTE_DOMAIN, + verse_id, + &route.authority_runtime_id, + &route.endpoint, + &protocols, + &priority, + &route.generation, + &certificate.provider_key.key_id, + &certificate.provider_key.x, + &certificate.provider_key.y, + &certificate.odin_key_id, + &issued, + &expires, + ])) +} + +pub fn canonical_session(request: &SessionOpen, endpoint: &str) -> Vec { + canonical_fields(&[ + SESSION_DOMAIN, + &request.client_nonce, + &request.message_id, + &request.source_runtime_id, + &request.verse_id, + &request.authority_runtime_id, + &request.protocol_id, + endpoint, + &request.route_generation, + ]) +} + +/// `(scheme, lowercase host)`, or `None` where `System.Uri` refuses the spelling. +/// A lenient parse could call a refused endpoint loopback or protected, so the +/// shape rule is the same strict one the TypeScript port applies. +fn split_endpoint(value: &str) -> Option<(String, String)> { + let raw = value.trim_matches(|c: char| c <= ' '); + if raw.contains('\\') { + return None; + } + let (scheme, rest) = raw.split_once("://")?; + let mut scheme_chars = scheme.chars(); + if !scheme_chars.next()?.is_ascii_alphabetic() + || !scheme_chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) + { + return None; + } + let authority = &rest[..rest.find(['/', '?', '#']).unwrap_or(rest.len())]; + if authority.is_empty() { + return None; + } + let after_user = &authority[authority.rfind('@').map_or(0, |i| i + 1)..]; + let host = if after_user.starts_with('[') { + &after_user[..after_user.find(']').map_or(after_user.len(), |i| i + 1)] + } else { + after_user.split(':').next().unwrap_or("") + }; + if host.is_empty() || host.chars().any(|c| !('\u{21}'..='\u{7e}').contains(&c)) || host.ends_with('.') { + return None; + } + Some(( + scheme.to_ascii_lowercase(), + host.trim_matches(['[', ']']).to_ascii_lowercase(), + )) +} + +pub fn is_loopback_endpoint(endpoint: &str) -> bool { + let Some((_, host)) = split_endpoint(endpoint) else { + return false; + }; + let octets: Vec<&str> = host.split('.').collect(); + matches!(host.as_str(), "localhost" | "loopback" | "::1" | "::ffff:127.0.0.1") + || (octets.len() == 4 + && octets[0] == "127" + && octets[1..].iter().all(|o| (1..=3).contains(&o.len()) && o.bytes().all(|b| b.is_ascii_digit()))) +} + +pub fn is_protected_endpoint(endpoint: &str) -> bool { + split_endpoint(endpoint) + .is_some_and(|(scheme, _)| scheme == "wss" || scheme == "https" || scheme.contains("quic")) +} + +/// Consumer-owned policy. Fails closed unless a route is valid under it. +pub struct AuthorityTrustPolicy { + mode: TrustMode, + roots: BTreeMap, +} + +impl AuthorityTrustPolicy { + pub fn new(mode: TrustMode, odin_roots: Vec) -> Result { + let mut roots = BTreeMap::new(); + for root in odin_roots { + let key_id = root.key_id.clone(); + if roots.insert(key_id.clone(), root).is_some() { + return Err(format!( + "Odin root key id '{key_id}' is listed more than once in the trust policy." + )); + } + } + Ok(Self { mode, roots }) + } + + pub fn validate(&self, verse_id: &str, route: &AuthorityRoute, now_unix_milliseconds: i64) -> Result<(), AuthorityRouteRefusal> { + self.verify(verse_id, route, now_unix_milliseconds).map(|_| ()) + } + + pub fn verify(&self, verse_id: &str, route: &AuthorityRoute, now_unix_milliseconds: i64) -> Result { + let refuse = |message: &str| AuthorityRouteRefusal { + message: message.to_owned(), + endpoint: route.endpoint.clone(), + }; + let local = self.mode == TrustMode::LocalDevelopment && is_loopback_endpoint(&route.endpoint); + let certificate = match &route.certificate { + Some(certificate) if !is_blank(&certificate.signature) => certificate, + _ if local => { + return Ok(VerifiedRoute { route: route.clone(), provider_key: None, is_local_development: true }); + } + _ => return Err(refuse("Remote CultMesh routes require an Odin-signed authority certificate.")), + }; + if !is_protected_endpoint(&route.endpoint) && !local { + return Err(refuse("Authenticated remote CultMesh routes require TLS or QUIC channel protection.")); + } + let Some(root) = self.roots.get(&certificate.odin_key_id) else { + return Err(refuse(&format!( + "Odin key '{}' is not trusted by this consumer.", + certificate.odin_key_id + ))); + }; + if now_unix_milliseconds < certificate.issued_at_unix_milliseconds + || now_unix_milliseconds >= certificate.expires_at_unix_milliseconds + { + return Err(refuse("The Odin route certificate is not currently valid.")); + } + let Some(signature) = base64_decode(&certificate.signature) else { + return Err(refuse("The Odin route signature is not base64.")); + }; + if signature.len() != 64 { + return Err(refuse("The Odin route signature is not IEEE P1363 P-256.")); + } + let canonical = canonical_route(verse_id, route).map_err(|message| refuse(&message))?; + if !root.verify(&canonical, &certificate.signature) { + return Err(refuse("The Odin route certificate signature is invalid.")); + } + Ok(VerifiedRoute { + route: route.clone(), + provider_key: Some(certificate.provider_key.clone()), + is_local_development: false, + }) + } +} + +/// Verifies the Odin route certificate (refusing) and the provider nonce proof. +pub fn verify_session_proof( + request: &SessionOpen, + response: &SessionAccepted, + verse_id: &str, + route: &AuthorityRoute, + trust: &AuthorityTrustPolicy, + now_unix_milliseconds: i64, +) -> Result { + let verified = trust.verify(verse_id, route, now_unix_milliseconds)?; + if verified.is_local_development { + return Ok(true); + } + let Some(key) = verified.provider_key else { + return Ok(false); + }; + if response.provider_key_id != key.key_id + || response.client_nonce != request.client_nonce + || is_blank(&response.provider_signature) + { + return Ok(false); + } + Ok(key.verify(&canonical_session(request, &route.endpoint), &response.provider_signature)) +} diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index e50719cd..5252fe09 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -25,6 +25,7 @@ use std::time::Duration; use std::time::Instant; mod rudp_document_server; +pub mod authority; pub use rudp_document_server::*; diff --git a/packages/cultmesh-rs/tests/authority_route_vectors.rs b/packages/cultmesh-rs/tests/authority_route_vectors.rs new file mode 100644 index 00000000..40cc5bba --- /dev/null +++ b/packages/cultmesh-rs/tests/authority_route_vectors.rs @@ -0,0 +1,261 @@ +//! Judges `contracts/cultmesh/authority-route-vectors*.json` in Rust. +//! +//! `authority-route-vectors.json` is written by the C# reference (its signatures +//! are captured C# bytes); the `.ts-signed.json` file is signed by TypeScript +//! over awkward inputs. Each negative case fails when the matching check is +//! removed from `cultmesh_rs::authority`. + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use cultmesh_rs::authority::*; +use p256::ecdsa::Signature; +use serde_json::Value; +use std::path::PathBuf; + +struct Vector { + now: i64, + root: P256PublicKey, + verse_id: String, + route: AuthorityRoute, + request: SessionOpen, + accepted: SessionAccepted, + trust: AuthorityTrustPolicy, +} + +fn s<'a>(value: &'a Value, key: &str) -> &'a str { + value[key].as_str().unwrap_or_else(|| panic!("missing string {key}")) +} + +fn key(value: &Value) -> P256PublicKey { + P256PublicKey::new(s(value, "keyId"), s(value, "x"), s(value, "y")).unwrap() +} + +fn load(name: &str) -> Vector { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../contracts/cultmesh") + .join(name); + let doc: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let r = &doc["route"]; + let c = &r["certificate"]; + let certificate = RouteCertificate::new( + key(&c["providerKey"]), + s(c, "odinKeyId"), + c["issuedAtUnixMilliseconds"].as_i64().unwrap(), + c["expiresAtUnixMilliseconds"].as_i64().unwrap(), + s(c, "signature"), + ) + .unwrap(); + let protocols: Vec<&str> = r["protocolIds"].as_array().unwrap().iter().map(|v| v.as_str().unwrap()).collect(); + let route = AuthorityRoute::new( + s(r, "authorityRuntimeId"), + s(r, "endpoint"), + &protocols, + r["priority"].as_i64().unwrap() as i32, + Some(s(r, "generation")), + Some(certificate), + ) + .unwrap(); + let p = &doc["sessionProof"]; + let q = &p["request"]; + let root = key(&doc["odinRoot"]); + Vector { + now: doc["nowUnixMilliseconds"].as_i64().unwrap(), + verse_id: s(r, "verseId").to_owned(), + route, + request: SessionOpen { + client_nonce: s(q, "clientNonce").into(), + message_id: s(q, "messageId").into(), + source_runtime_id: s(q, "sourceRuntimeId").into(), + verse_id: s(q, "verseId").into(), + authority_runtime_id: s(q, "authorityRuntimeId").into(), + protocol_id: s(q, "protocolId").into(), + route_generation: s(q, "routeGeneration").into(), + }, + accepted: SessionAccepted { + client_nonce: s(q, "clientNonce").into(), + provider_key_id: s(p, "providerKeyId").into(), + provider_signature: s(p, "providerSignature").into(), + }, + trust: AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![root.clone()]).unwrap(), + root, + } +} + +impl Vector { + fn certificate(&self) -> &RouteCertificate { + self.route.certificate.as_ref().unwrap() + } + fn with_certificate(&self, edit: impl FnOnce(&mut RouteCertificate)) -> AuthorityRoute { + let mut route = self.route.clone(); + edit(route.certificate.as_mut().unwrap()); + route + } + fn refusal(&self, route: &AuthorityRoute, now: i64) -> String { + let error = self.trust.validate(&self.verse_id, route, now).expect_err("route must be refused"); + assert_eq!(AuthorityRouteRefusal::REASON, "authentication"); + assert_eq!(route.endpoint, error.endpoint); + error.message + } + fn proof(&self, request: &SessionOpen, accepted: &SessionAccepted) -> bool { + verify_session_proof(request, accepted, &self.verse_id, &self.route, &self.trust, self.now).unwrap() + } +} + +const INVALID: &str = "The Odin route certificate signature is invalid."; + +#[test] +fn cs_written_vector_verifies_route_and_session_proof() { + let v = load("authority-route-vectors.json"); + v.trust.validate(&v.verse_id, &v.route, v.now).unwrap(); + assert!(v.proof(&v.request, &v.accepted)); +} + +#[test] +fn ts_signed_vector_verifies_and_tamper_is_refused() { + let v = load("authority-route-vectors.ts-signed.json"); + v.trust.validate(&v.verse_id, &v.route, v.now).unwrap(); + assert!(v.proof(&v.request, &v.accepted)); + let mut tampered = v.route.clone(); + tampered.priority += 1; + assert_eq!(INVALID, v.refusal(&tampered, v.now)); +} + +#[test] +fn canonical_route_is_length_prefixed_and_sorted() { + let v = load("authority-route-vectors.json"); + let bytes = canonical_route(&v.verse_id, &v.route).unwrap(); + assert!(bytes.starts_with(b"\x00\x00\x00\x26gamecult.cultmesh.route-certificate.v1\x00\x00\x00\x08aetheria")); + let needle = b"cultmesh.content.v1\x1fcultmesh.documents.v1"; + assert!(bytes.windows(needle.len()).any(|w| w == needle)); +} + +#[test] +fn tampered_payload_fields_are_refused() { + let v = load("authority-route-vectors.json"); + let edits: Vec> = vec![ + Box::new(|r| r.priority = 8), + Box::new(|r| r.endpoint = "wss://evil.example/mesh".into()), + Box::new(|r| r.protocol_ids = vec!["cultmesh.documents.v1".into()]), + Box::new(|r| r.generation = "other".into()), + Box::new(|r| r.certificate.as_mut().unwrap().expires_at_unix_milliseconds += 1), + ]; + for edit in edits { + let mut route = v.route.clone(); + edit(&mut route); + assert_eq!(INVALID, v.refusal(&route, v.now)); + } + let error = v.trust.validate("other-verse", &v.route, v.now).unwrap_err(); + assert_eq!(INVALID, error.message); +} + +#[test] +fn wrong_root_key_is_refused() { + let v = load("authority-route-vectors.json"); + // The provider's key is a real P-256 key that did not sign this route. + let provider = &v.certificate().provider_key; + let impostor = P256PublicKey::new(&v.root.key_id, &provider.x, &provider.y).unwrap(); + let trust = AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![impostor]).unwrap(); + assert_eq!(INVALID, trust.validate(&v.verse_id, &v.route, v.now).unwrap_err().message); +} + +#[test] +fn untrusted_odin_key_id_is_refused() { + let v = load("authority-route-vectors.json"); + let other = P256PublicKey::new("someone-else", &v.root.x, &v.root.y).unwrap(); + let trust = AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![other]).unwrap(); + assert_eq!( + "Odin key 'odin-vector' is not trusted by this consumer.", + trust.validate(&v.verse_id, &v.route, v.now).unwrap_err().message + ); +} + +#[test] +fn expired_and_not_yet_valid_are_refused() { + let v = load("authority-route-vectors.json"); + let window = "The Odin route certificate is not currently valid."; + let (issued, expires) = (v.certificate().issued_at_unix_milliseconds, v.certificate().expires_at_unix_milliseconds); + v.trust.validate(&v.verse_id, &v.route, expires - 1).unwrap(); + v.trust.validate(&v.verse_id, &v.route, issued).unwrap(); + assert_eq!(window, v.refusal(&v.route, expires)); + assert_eq!(window, v.refusal(&v.route, issued - 1)); +} + +#[test] +fn unsigned_and_unprotected_routes_are_refused() { + let v = load("authority-route-vectors.json"); + let unsigned_message = "Remote CultMesh routes require an Odin-signed authority certificate."; + let mut unsigned = v.route.clone(); + unsigned.certificate = None; + assert_eq!(unsigned_message, v.refusal(&unsigned, v.now)); + let blank = v.with_certificate(|c| c.signature = " ".into()); + assert_eq!(unsigned_message, v.refusal(&blank, v.now)); + let mut plain = v.route.clone(); + plain.endpoint = "ws://provider.example/mesh".into(); + assert_eq!( + "Authenticated remote CultMesh routes require TLS or QUIC channel protection.", + v.refusal(&plain, v.now) + ); +} + +#[test] +fn local_development_admits_only_unsigned_loopback() { + let v = load("authority-route-vectors.json"); + let dev = AuthorityTrustPolicy::new(TrustMode::LocalDevelopment, vec![]).unwrap(); + let route = |endpoint: &str| AuthorityRoute::new("a", endpoint, &[], 0, None, None).unwrap(); + let loopback = route("ws://127.0.0.1:9/mesh"); + dev.validate(&v.verse_id, &loopback, v.now).unwrap(); + assert!(verify_session_proof(&v.request, &v.accepted, &v.verse_id, &loopback, &dev, v.now).unwrap()); + for endpoint in ["ws://provider.example/mesh", "ws://127.0.0.1./mesh", "ws:127.0.0.1"] { + assert!(dev.validate(&v.verse_id, &route(endpoint), v.now).is_err(), "{endpoint}"); + } + let remote = AuthorityTrustPolicy::new(TrustMode::AuthenticatedRemote, vec![]).unwrap(); + assert!(remote.validate(&v.verse_id, &loopback, v.now).is_err()); +} + +#[test] +fn malformed_signatures_get_their_own_refusals() { + let v = load("authority-route-vectors.json"); + let not_base64 = v.with_certificate(|c| c.signature = "!!not base64!!".into()); + assert_eq!("The Odin route signature is not base64.", v.refusal(¬_base64, v.now)); + let short = v.with_certificate(|c| c.signature = STANDARD.encode([b'x'; 63])); + assert_eq!("The Odin route signature is not IEEE P1363 P-256.", v.refusal(&short, v.now)); +} + +#[test] +fn high_s_signature_verifies_as_it_does_in_dotnet() { + let v = load("authority-route-vectors.json"); + let raw = STANDARD.decode(&v.certificate().signature).unwrap(); + let signature = Signature::from_slice(&raw).unwrap(); + let flipped = Signature::from_scalars(*signature.r().as_ref(), -*signature.s().as_ref()).unwrap(); + let high = v.with_certificate(|c| c.signature = STANDARD.encode(flipped.to_bytes())); + v.trust.validate(&v.verse_id, &high, v.now).unwrap(); +} + +#[test] +fn session_proof_rejects_replay_wrong_key_and_forgery() { + let v = load("authority-route-vectors.json"); + let mut replayed = v.request.clone(); + replayed.client_nonce = "replayed".into(); + let mut accepted = v.accepted.clone(); + accepted.client_nonce = "replayed".into(); + assert!(!v.proof(&replayed, &accepted)); + let mut wrong_id = v.accepted.clone(); + wrong_id.provider_key_id = "other".into(); + assert!(!v.proof(&v.request, &wrong_id)); + let mut wrong_nonce = v.accepted.clone(); + wrong_nonce.client_nonce = "other".into(); + assert!(!v.proof(&v.request, &wrong_nonce)); + let mut empty = v.accepted.clone(); + empty.provider_signature = String::new(); + assert!(!v.proof(&v.request, &empty)); + let mut raw = STANDARD.decode(&v.accepted.provider_signature).unwrap(); + raw[0] ^= 1; + let mut forged = v.accepted.clone(); + forged.provider_signature = STANDARD.encode(raw); + assert!(!v.proof(&v.request, &forged)); + let mut moved = v.request.clone(); + moved.protocol_id = "cultmesh.content.v1".into(); + assert!(!v.proof(&moved, &v.accepted)); + let expires = v.certificate().expires_at_unix_milliseconds; + assert!(verify_session_proof(&v.request, &v.accepted, &v.verse_id, &v.route, &v.trust, expires).is_err()); +} diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index cca85a79..04df5ddb 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per + remote endpoint, ported from the C# `CultNetRudpSocketTransportServer` and held to its bytes by + `tests/vectors/rudp-multipeer-server/transcript.txt`. The caller drives it (`try_receive_once`, `poll_resends`). Like the C# + listener it never reaps idle peers. `CultNetRudpSession` gained `next_sequence`, `withdraw_send` and `end_with`, and + `CultNetTransportStats` gained `send_failures`. + - Removed `resolve_document_and_schema_id_for_raw_record` from the package exports. A raw record resolves to one local schema id, the one its document registers, so there is no wire id to return. Use `resolve_document_for_raw_record(documents_by_schema_id, schema_id, record)`, which returns the document; take the schema id from `document.catalog_entry().schema_id`. diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index 9bad58f6..3b05079c 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -106,6 +106,12 @@ from .shard_replication import ( ) from .snapshot import CultNetRawDocumentRecord, CultNetRawSnapshotResponse from .subscription import CultNetDatabaseChange +from .rudp_server import ( + CultNetRudpSocketServerFrame, + CultNetRudpSocketServerOptions, + CultNetRudpSocketServerPeer, + CultNetRudpSocketTransportServer, +) from .transport import ( CultNetRudpDeliveredFrame, CultNetRudpPacket, @@ -215,6 +221,10 @@ __all__ = [ "CultNetRudpReceiveResult", "CultNetRudpSendOptions", "CultNetRudpSession", + "CultNetRudpSocketServerFrame", + "CultNetRudpSocketServerOptions", + "CultNetRudpSocketServerPeer", + "CultNetRudpSocketTransportServer", "is_permanent_send_error", "send_error_code", "CultNetRudpSessionOptions", diff --git a/packages/cultnet-py/src/cultnet_py/rudp_server.py b/packages/cultnet-py/src/cultnet_py/rudp_server.py new file mode 100644 index 00000000..2b162043 --- /dev/null +++ b/packages/cultnet-py/src/cultnet_py/rudp_server.py @@ -0,0 +1,345 @@ +"""Multi-peer RUDP listener: one UDP socket, one reliable session per remote endpoint. + +Matches the C# `CultNetRudpSocketTransportServer` (the reference): a Connect opens a peer's +session, a Connect from the same endpoint that is not a repeat replaces only that peer, and +a peer that ends, is refused or cannot be answered never disturbs the others. The caller +drives it: `try_receive_once` consumes one datagram, `poll_resends` sends what is due. +Everything runs on the caller's thread, so no session needs a lock. +""" +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +import socket + +from .transport import ( + REFUSED_PACKET_REASON, + CultNetRudpPacket, + CultNetRudpPacketType, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetReconnectPolicy, + CultNetTransportFrame, + CultNetTransportStats, + _channel_send_options, + _now_ms, + create_rudp_transport_profile, + decode_rudp_packet, + encode_rudp_packet, + is_permanent_send_error, +) + +REPLACED_PEER_REASON = b"replaced by a new Connect generation" + + +@dataclass +class CultNetRudpSocketServerOptions: + runtime_id: str + socket: socket.socket + connection_id: int + initial_sequence: int | None = None + resend_delay_ms: int = 250 + transport_id: str = "rudp" + max_payload_bytes: int | None = None + max_fragment_bytes: int | None = None + max_pending_reliable_packets: int | None = None + reconnect_policy: CultNetReconnectPolicy | None = None + accept_payload: bytes = b"" + + +class CultNetRudpSocketServerPeer: + """One remote endpoint's session. A peer that reconnects from the same endpoint is a new object.""" + + def __init__(self, remote_addr: tuple, connect_payload: bytes, session: CultNetRudpSession) -> None: + self.remote_addr = remote_addr + self.connect_payload = connect_payload + self.session = session + # The last transport-level reason the session ended with, once it has. + self.disconnect_reason: bytes | None = None + + @property + def connected(self) -> bool: + return self.session.connected + + @property + def pending_reliable_packet_count(self) -> int: + return self.session.outstanding_reliable_packet_count + + +@dataclass(frozen=True) +class CultNetRudpSocketServerFrame: + peer: CultNetRudpSocketServerPeer + frame: CultNetTransportFrame + + +def _remote_key(addr: tuple) -> tuple: + return tuple(addr[:2]) + + +class CultNetRudpSocketTransportServer: + def __init__(self, options: CultNetRudpSocketServerOptions) -> None: + if not options.runtime_id.strip(): + raise ValueError("Runtime id is required.") + self.socket = options.socket + self._options = options + host, port = self.socket.getsockname()[:2] + self.profile = create_rudp_transport_profile( + options.runtime_id, + transport_id=options.transport_id, + host=host, + port=port, + max_payload_bytes=options.max_payload_bytes, + max_fragment_bytes=options.max_fragment_bytes, + max_pending_reliable_packets=options.max_pending_reliable_packets, + reconnect_policy=options.reconnect_policy, + ) + self._peers: dict[tuple, CultNetRudpSocketServerPeer] = {} + self._delivered: deque[CultNetRudpSocketServerFrame] = deque() + self._bytes_received = 0 + self._bytes_sent = 0 + self._frames_received = 0 + self._frames_sent = 0 + self._packets_dropped = 0 + self._send_failures = 0 + self._closed = False + self.peer_disconnected: list = [] # callbacks taking the peer + + @property + def peers(self) -> tuple[CultNetRudpSocketServerPeer, ...]: + return tuple(self._peers.values()) + + @property + def stats(self) -> CultNetTransportStats: + return CultNetTransportStats( + bytes_received=self._bytes_received, + bytes_sent=self._bytes_sent, + frames_received=self._frames_received, + frames_sent=self._frames_sent, + packets_dropped=self._packets_dropped, + send_failures=self._send_failures, + ) + + def send(self, peer: CultNetRudpSocketServerPeer, channel_id: str, payload: bytes) -> None: + """Sends to one peer. Raises when that peer's session has ended or was replaced, or when a + datagram can never be sent as built: the first fragment failing queues nothing, a later one + ends the session, as the peer has seen part of the frame.""" + if self._peers.get(_remote_key(peer.remote_addr)) is not peer: + raise ConnectionError("RUDP peer is not connected.") + first_sequence = peer.session.next_sequence + packets = peer.session.send_many( + channel_id, + payload, + _channel_send_options(channel_id, _now_ms()), + max_fragment_bytes=self._options.max_fragment_bytes, + ) + for index, packet in enumerate(packets): + error = self._send_packet(peer.remote_addr, packet) + if error is None: + continue + if index == 0: + peer.session.withdraw_send(first_sequence) + else: + self._end_unsendable_peer(peer, error) + raise error + self._frames_sent += 1 + + def send_schema(self, peer: CultNetRudpSocketServerPeer, payload: bytes) -> None: + self.send(peer, "schema", payload) + + def broadcast(self, channel_id: str, payload: bytes) -> list[CultNetRudpSocketServerPeer]: + """Sends to every connected peer and returns the peers that could not be sent to; one + peer's failure never stops the others.""" + failed = [] + for peer in self.peers: + try: + self.send(peer, channel_id, payload) + except (OSError, ConnectionError, ValueError): + failed.append(peer) + return failed + + def disconnect(self, peer: CultNetRudpSocketServerPeer, reason: bytes = b"") -> bool: + """Ends one peer's session, telling it why. False when the session had already ended.""" + return self.end_peer_session(peer, reason) + + def end_peer_session(self, peer: CultNetRudpSocketServerPeer, reason: bytes) -> bool: + """Ends the session of a peer whose delivered frame the application could not decode or + handle, with a goodbye carrying the application's reason. The frame is already + acknowledged, so the session cannot be kept; the peer's frames still queued are not + delivered. A session that already ended owes no goodbye, since the endpoint may belong + to a new session; the call then returns False.""" + self._packets_dropped += 1 + if peer.session.ended: + return False + self._send_packet(peer.remote_addr, peer.session.end_with(reason)) + self._delivered = deque(item for item in self._delivered if item.peer is not peer) + self._retire(peer, reason) + return True + + def receive_once(self) -> CultNetRudpSocketServerFrame | None: + return self.try_receive_once()[1] + + def try_receive_once(self) -> tuple[bool, CultNetRudpSocketServerFrame | None]: + """Consumes one transport work item. Returns whether progress was made, independently of + whether it completed an application frame.""" + if self._delivered: + return True, self._delivered.popleft() + try: + wire, remote = self.socket.recvfrom(65535) + except (TimeoutError, BlockingIOError, ConnectionResetError): + return False, None + self._bytes_received += len(wire) + + try: + packet = decode_rudp_packet(wire) + except ValueError: + self._packets_dropped += 1 + return True, None + if packet.connection_id != self._options.connection_id: + self._packets_dropped += 1 + return True, None + + key = _remote_key(remote) + if packet.packet_type == CultNetRudpPacketType.CONNECT: + self._receive_connect(packet, remote, key) + return True, None + peer = self._peers.get(key) + if peer is None: + self._packets_dropped += 1 + return True, None + self._receive_from_peer(peer, key, packet) + return True, (self._delivered.popleft() if self._delivered else None) + + def poll_resends(self) -> None: + """Sends the reliable packets whose resend timers are due, for every peer.""" + for peer in self.peers: + if self._peers.get(_remote_key(peer.remote_addr)) is not peer: + continue + for packet in peer.session.due_resends(_now_ms()): + error = self._send_packet(peer.remote_addr, packet) + if error is not None: + self._end_unsendable_peer(peer, error) + break + + def close(self) -> None: + if self._closed: + return + self._closed = True + self.socket.close() + + def __enter__(self) -> "CultNetRudpSocketTransportServer": + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def _receive_connect(self, packet: CultNetRudpPacket, remote: tuple, key: tuple) -> None: + admitted = self._peers.get(key) + accept_payload = self._options.accept_payload + # The admitted session decides whether this Connect repeats the one it accepted: a repeat + # gets the Accept already owed and queues nothing. Any other Connect is a new generation + # and ends the admitted session here. + if admitted is not None: + if admitted.session.connect_repeats(packet): + try: + reply = admitted.session.accept_connect(packet, _now_ms(), accept_payload) + except ValueError: + self._packets_dropped += 1 + return + error = self._send_packet(admitted.remote_addr, reply) + if error is not None: + self._end_unsendable_peer(admitted, error) + return + admitted.session.reset_peer_state() + + peer = CultNetRudpSocketServerPeer( + tuple(remote), + packet.payload, + CultNetRudpSession( + CultNetRudpSessionOptions( + connection_id=self._options.connection_id, + initial_sequence=self._options.initial_sequence, + resend_delay_ms=self._options.resend_delay_ms, + max_pending_reliable_packets=self._options.max_pending_reliable_packets, + ) + ), + ) + try: + accept = peer.session.accept_connect(packet, _now_ms(), accept_payload) + except ValueError: + self._packets_dropped += 1 + accept = None + # A peer that cannot be answered gets no session, and the one it replaced has already ended. + if accept is None or self._send_packet(peer.remote_addr, accept) is not None: + self._peers.pop(key, None) + if accept is not None: + self._packets_dropped += 1 + else: + self._peers[key] = peer + if admitted is not None: + admitted.disconnect_reason = REPLACED_PEER_REASON + self._notify_disconnected(admitted) + + def _receive_from_peer(self, peer: CultNetRudpSocketServerPeer, key: tuple, packet: CultNetRudpPacket) -> None: + session = peer.session + try: + result = session.receive(packet, _now_ms()) + except ValueError: + # receive() has already recorded the packet's reliable sequence, so the session cannot + # be kept: a retransmit would be acknowledged and the frame silently lost. + self._packets_dropped += 1 + self._send_packet(peer.remote_addr, session.end_with(REFUSED_PACKET_REASON)) + self._retire(peer, REFUSED_PACKET_REASON) + return + error = None + if result.reply is not None: + error = self._send_packet(peer.remote_addr, result.reply) + for ready in result.ready_to_send: + if error is None: + error = self._send_packet(peer.remote_addr, ready) + if result.disconnected: + self._retire(peer, result.disconnect_reason) + return + for frame in result.delivered: + self._delivered.append( + CultNetRudpSocketServerFrame(peer, CultNetTransportFrame(frame.channel_id, frame.payload)) + ) + self._frames_received += 1 + if error is None and packet.packet_type == CultNetRudpPacketType.DATA: + error = self._send_packet(peer.remote_addr, session.create_ack_for_received(packet.sequence)) + if error is not None: + self._end_unsendable_peer(peer, error) + + def _end_unsendable_peer(self, peer: CultNetRudpSocketServerPeer, error: OSError) -> None: + """An admitted peer owes a packet that can never be sent as built: its session ends, never + the poll, the peer is told, and the disconnect reason names the error. A session that has + already ended owes no goodbye, since the endpoint may belong to a new session.""" + if peer.session.ended: + return + goodbye = peer.session.end_unsendable_session(error) + self._send_packet(peer.remote_addr, goodbye) + self._retire(peer, goodbye.payload) + + def _retire(self, peer: CultNetRudpSocketServerPeer, reason: bytes | None) -> None: + # A peer a new Connect already replaced was reported then; its key now names the successor. + key = _remote_key(peer.remote_addr) + if self._peers.get(key) is not peer: + return + del self._peers[key] + peer.disconnect_reason = reason + self._notify_disconnected(peer) + + def _notify_disconnected(self, peer: CultNetRudpSocketServerPeer) -> None: + for callback in list(self.peer_disconnected): + callback(peer) + + def _send_packet(self, remote_addr: tuple, packet: CultNetRudpPacket) -> OSError | None: + """A transient failure is that peer's lost datagram: counted, never raised. A permanent + one (`is_permanent_send_error`) is returned for the caller to refuse or end the session.""" + wire = encode_rudp_packet(packet) + try: + self._bytes_sent += self.socket.sendto(wire, remote_addr) + except OSError as error: + if is_permanent_send_error(error): + return error + self._send_failures += 1 + return None diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 76b38bf5..3ecfa31b 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -29,6 +29,9 @@ class CultNetTransportStats: # transport: malformed frames, another session's connection id, a sender # that is not the peer, or a packet the session refuses. packets_dropped: int = 0 + # Datagrams a multi-peer server could not send to one peer and lost: each is + # a transient failure, never a reason to end a session. + send_failures: int = 0 @dataclass(frozen=True) @@ -437,6 +440,7 @@ def create_rudp_transport_profile( UNSENDABLE_PACKET_REASON = "packet could not be sent" +REFUSED_PACKET_REASON = b"session refused a packet" # The fixed name of each permanent send failure, keyed by errno, Winsock equivalents included. _PERMANENT_SEND_ERROR_CODES = { @@ -600,6 +604,28 @@ class CultNetRudpSession: self._fragment_buffers.clear() self._fragment_sets_evicted = 0 + @property + def next_sequence(self) -> int: + return self._next_sequence + + def withdraw_send(self, first_sequence: int) -> None: + """Undoes a send that never reached the wire: every packet it created at or after + `first_sequence`, pending or queued, leaves no trace and its sequences are reissued, + so the peer never sees a gap. Only valid while nothing else has been sent since.""" + for sequence in [s for s in self._pending_reliable if s >= first_sequence]: + del self._pending_reliable[sequence] + kept = [p for p in self._queued_reliable if p.sequence < first_sequence] + self._queued_reliable.clear() + self._queued_reliable.extend(kept) + self._next_sequence = first_sequence + + def end_with(self, reason: bytes) -> CultNetRudpPacket: + """Ends a session whose peer's frame was refused or failed after the session acknowledged + it. The peer's sequences are forgotten before the goodbye is built, or its ack field would + acknowledge the very frame that was refused.""" + self.reset_peer_state() + return self.create_disconnect(reason) + def create_connect(self, now_ms: int = 0, payload: bytes = b"") -> CultNetRudpPacket: # A session that has had a peer starts a new generation: nothing it # learned from that peer describes the one this Connect reaches, and @@ -1397,11 +1423,8 @@ class CultNetRudpSocketTransportConnection: self.socket.close() def _end_refused_session(self) -> None: - reason = b"session refused a packet" - # The goodbye is built after the reset, or its ack field would acknowledge - # the very frame the session refused. - self.session.reset_peer_state() - goodbye = self.session.create_disconnect(reason) + reason = REFUSED_PACKET_REASON + goodbye = self.session.end_with(reason) self.disconnect_reason = reason try: self._send_packet(goodbye) diff --git a/packages/cultnet-py/tests/test_rudp_multipeer_server.py b/packages/cultnet-py/tests/test_rudp_multipeer_server.py new file mode 100644 index 00000000..3bb14640 --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_multipeer_server.py @@ -0,0 +1,350 @@ +"""One UDP socket serves many peers, each in its own session, as the C# reference does. + +Sockets bind port 0, so no test races another for a port. +""" +from __future__ import annotations + +import errno +import socket +import time +import unittest +from pathlib import Path + +from cultnet_py.rudp_server import ( + CultNetRudpSocketServerOptions, + CultNetRudpSocketTransportServer, +) +from cultnet_py.transport import ( + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + decode_rudp_packet, + encode_rudp_packet, +) + +CONNECTION_ID = 0x43554C54 +VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "rudp-multipeer-server" / "transcript.txt" + + +def bind() -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + sock.settimeout(0.03) + return sock + + +def make_server(**kwargs) -> CultNetRudpSocketTransportServer: + options = dict(runtime_id="py-test", socket=bind(), connection_id=CONNECTION_ID, resend_delay_ms=5) + options.update(kwargs) + return CultNetRudpSocketTransportServer(CultNetRudpSocketServerOptions(**options)) + + +def pump(server: CultNetRudpSocketTransportServer) -> list: + """Consumes datagrams until the socket is idle, returning the frames delivered.""" + frames = [] + while True: + progressed, frame = server.try_receive_once() + if frame is not None: + frames.append(frame) + if not progressed: + return frames + + +def drain(sock: socket.socket) -> list[bytes]: + wires = [] + while True: + try: + wires.append(sock.recvfrom(65535)[0]) + except TimeoutError: + return wires + + +class Client: + def __init__(self, server: CultNetRudpSocketTransportServer, initial_sequence: int) -> None: + self.server = server + self.socket = bind() + self.session = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=initial_sequence) + ) + + @property + def addr(self) -> tuple: + return self.socket.getsockname() + + def transmit(self, packet) -> None: + self.socket.sendto(encode_rudp_packet(packet), self.server.socket.getsockname()) + + def connect(self, payload: bytes = b"") -> None: + self.transmit(self.session.create_connect(0, payload)) + pump(self.server) + self.receive() + + def receive(self) -> list: + packets = [decode_rudp_packet(wire) for wire in drain(self.socket)] + for packet in packets: + if self.session.connected or packet.packet_type == CultNetRudpPacketType.ACCEPT: + self.session.receive(packet, 0) + return packets + + def write(self, text: str) -> None: + self.transmit( + self.session.send("schema", text.encode(), CultNetRudpSendOptions(reliable=True, ordered=True)) + ) + + def peer(self): + return next(p for p in self.server.peers if p.remote_addr == self.addr) + + def close(self) -> None: + self.socket.close() + + +class MultiPeerServerTests(unittest.TestCase): + def setUp(self) -> None: + self.server = make_server(initial_sequence=500, accept_payload=b"acc") + self.addCleanup(self.server.close) + self.clients: list[Client] = [] + + def client(self, initial_sequence: int) -> Client: + client = Client(self.server, initial_sequence) + self.clients.append(client) + self.addCleanup(client.close) + return client + + def test_interleaved_peers_each_get_their_own_session_and_frames(self) -> None: + x, y = self.client(10), self.client(20) + x.connect(b"hx") + y.connect(b"hy") + self.assertEqual(len(self.server.peers), 2) + self.assertEqual(x.peer().connect_payload, b"hx") + self.assertEqual(y.peer().connect_payload, b"hy") + + x.write("x1") + y.write("y1") + x.write("x2") + y.write("y2") + frames = pump(self.server) + by_peer = {} + for item in frames: + by_peer.setdefault(item.peer.remote_addr, []).append(item.frame.payload) + self.assertEqual(by_peer, {x.addr: [b"x1", b"x2"], y.addr: [b"y1", b"y2"]}) + + self.server.send(x.peer(), "schema", b"to-x") + self.server.send(y.peer(), "schema", b"to-y") + for client, expected in ((x, b"to-x"), (y, b"to-y")): + data = [p for p in client.receive() if p.packet_type == CultNetRudpPacketType.DATA] + self.assertEqual([p.payload for p in data], [expected]) + + def test_one_peer_ending_does_not_disturb_the_other(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + y_peer = y.peer() + ended = [] + self.server.peer_disconnected.append(ended.append) + + x.transmit(x.session.create_disconnect(b"bye")) + pump(self.server) + self.assertEqual(len(self.server.peers), 1) + self.assertEqual([p.disconnect_reason for p in ended], [b"bye"]) + self.assertIs(y.peer(), y_peer) + + # Datagrams from the ended endpoint belong to no session. + before = self.server.stats.packets_dropped + x_late = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=99)) + x_late.accept_connect(x.session.create_connect(0), 0) + x.transmit(x_late.send("schema", b"late", CultNetRudpSendOptions(reliable=True, ordered=True))) + pump(self.server) + self.assertEqual(self.server.stats.packets_dropped, before + 1) + + y.write("still here") + self.assertEqual([f.frame.payload for f in pump(self.server)], [b"still here"]) + self.server.send(y_peer, "schema", b"reply") + self.assertEqual( + [p.payload for p in y.receive() if p.packet_type == CultNetRudpPacketType.DATA], [b"reply"] + ) + + def test_a_new_connect_from_the_same_endpoint_replaces_only_that_peer(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + old_y = y.peer() + ended = [] + self.server.peer_disconnected.append(ended.append) + + y.session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=30)) + y.connect(b"again") + self.assertEqual(ended, [old_y]) + self.assertEqual(old_y.disconnect_reason, b"replaced by a new Connect generation") + self.assertIsNot(y.peer(), old_y) + self.assertEqual(y.peer().connect_payload, b"again") + with self.assertRaises(ConnectionError): + self.server.send(old_y, "schema", b"to the old session") + self.assertEqual(len(self.server.peers), 2) + + def test_a_repeated_connect_is_answered_with_the_owed_accept_and_starts_nothing(self) -> None: + x = self.client(10) + connect = x.session.create_connect(0, b"h") + x.transmit(connect) + pump(self.server) + first = x.peer() + x.transmit(connect) + pump(self.server) + self.assertIs(x.peer(), first) + accepts = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.ACCEPT] + self.assertEqual(len(accepts), 2) + self.assertEqual(accepts[0].sequence, accepts[1].sequence) + + def test_strays_are_dropped_and_counted(self) -> None: + x = self.client(10) + x.socket.sendto(b"not rudp", self.server.socket.getsockname()) + other = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID + 1, initial_sequence=1)) + x.transmit(other.create_connect(0)) + x.transmit( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=1)).create_ping() + ) + pump(self.server) + self.assertEqual(self.server.stats.packets_dropped, 3) + self.assertEqual(self.server.peers, ()) + + def test_resends_go_only_to_peers_with_unacknowledged_writes(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + self.server.send(x.peer(), "schema", b"owed") + x.receive() # the first transmission is lost to the test: x never acks + y.receive() + time.sleep(0.02) + self.server.poll_resends() + x_data = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.DATA] + self.assertEqual([p.payload for p in x_data], [b"owed"]) + self.assertEqual([p for p in y.receive() if p.packet_type == CultNetRudpPacketType.DATA], []) + + def test_broadcast_reaches_every_peer(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + self.assertEqual(self.server.broadcast("schema", b"all"), []) + for client in (x, y): + self.assertEqual( + [p.payload for p in client.receive() if p.packet_type == CultNetRudpPacketType.DATA], [b"all"] + ) + + def test_an_application_ending_a_session_tells_the_peer_and_drops_its_queued_frames(self) -> None: + x, y = self.client(10), self.client(20) + x.connect() + y.connect() + x.write("x1") + y.write("y1") + # Both frames are decoded into the server's queue; ending x discards x's. + progressed, first = self.server.try_receive_once() + self.assertTrue(progressed) + pump_rest = pump(self.server) + frames = [first] + pump_rest if first else pump_rest + peer_x = x.peer() + self.assertTrue(self.server.end_peer_session(peer_x, b"server could not handle a message")) + self.assertFalse(self.server.end_peer_session(peer_x, b"again"), "an ended session owes no second goodbye") + goodbye = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.DISCONNECT] + self.assertEqual([p.payload for p in goodbye], [b"server could not handle a message"]) + self.assertEqual(len(self.server.peers), 1) + self.assertEqual({f.peer.remote_addr for f in frames}, {x.addr, y.addr}) + + def test_a_datagram_too_large_to_send_is_that_callers_error_and_queues_nothing(self) -> None: + x = self.client(10) + x.connect() + peer = x.peer() + owed = peer.pending_reliable_packet_count # the Accept, not yet acknowledged + with self.assertRaises(OSError) as raised: + self.server.send(peer, "schema", bytes(70_000)) + self.assertEqual(raised.exception.errno, errno.EMSGSIZE) + self.assertEqual(peer.pending_reliable_packet_count, owed) + self.assertEqual(len(self.server.peers), 1) + # The refused write left no gap in the sequence space the peer is waiting on. + self.server.send(peer, "schema", b"fine") + data = [p for p in x.receive() if p.packet_type == CultNetRudpPacketType.DATA] + self.assertEqual([(p.sequence, p.payload) for p in data], [(501, b"fine")]) + + +class UnsendableAfterFirstFragmentSocket: + """Delegates to a real socket; fails permanently once its budget of sends to one address is spent.""" + + def __init__(self, real: socket.socket, addr: tuple, budget: int) -> None: + self._real, self._addr, self._budget = real, addr, budget + + def sendto(self, wire: bytes, addr: tuple) -> int: + if tuple(addr) == self._addr: + if self._budget == 0: + raise OSError(errno.EMSGSIZE, "Message too long") + self._budget -= 1 + return self._real.sendto(wire, addr) + + def __getattr__(self, name: str): + return getattr(self._real, name) + + +class UnsendableSessionTests(unittest.TestCase): + def test_a_later_fragment_that_cannot_be_sent_ends_that_session_only(self) -> None: + server = make_server(initial_sequence=500, max_fragment_bytes=8) + self.addCleanup(server.close) + x, y = Client(server, 10), Client(server, 20) + self.addCleanup(x.close) + self.addCleanup(y.close) + x.connect() + y.connect() + x_peer = x.peer() + server.socket = UnsendableAfterFirstFragmentSocket(server.socket, x.addr, budget=1) + ended = [] + server.peer_disconnected.append(ended.append) + + with self.assertRaises(OSError): + server.send(x_peer, "schema", bytes(24)) + self.assertEqual(ended, [x_peer]) + self.assertEqual(x_peer.disconnect_reason, b"packet could not be sent: EMSGSIZE") + self.assertEqual(len(server.peers), 1) + # The goodbye could not be sent either (same address), so the peer is only retired; y is unharmed. + server.send(y.peer(), "schema", b"ok") + self.assertEqual( + [p.payload for p in y.receive() if p.packet_type == CultNetRudpPacketType.DATA], [b"ok"] + ) + + +class ReferenceTranscriptTests(unittest.TestCase): + """Replays the datagrams the C# reference listener was captured answering, byte for byte.""" + + def test_the_listener_answers_with_the_bytes_the_reference_does(self) -> None: + server = make_server( + connection_id=0x43554C54, initial_sequence=500, max_fragment_bytes=8, accept_payload=b"acc", resend_delay_ms=250 + ) + self.addCleanup(server.close) + sockets = {label: bind() for label in ("X", "Y")} + for sock in sockets.values(): + self.addCleanup(sock.close) + target = server.socket.getsockname() + + steps: list[tuple[list[str], list[tuple[str, str]]]] = [] + for line in VECTORS.read_text().splitlines(): + if line.startswith("#") or not line.strip(): + continue + fields = line.split(" ") + if fields[0] == "out": + steps[-1][1].append((fields[1], fields[2])) + else: + steps.append((fields, [])) + self.assertGreater(len(steps), 10) + + for action, expected in steps: + if action[0] == "in": + sockets[action[1]].sendto(bytes.fromhex(action[2]), target) + while not server.try_receive_once()[0]: + time.sleep(0.001) + else: + peer = next(p for p in server.peers if p.remote_addr == sockets[action[1]].getsockname()) + server.send(peer, action[2], bytes.fromhex(action[3])) + actual = [(label, wire.hex()) for label in ("X", "Y") for wire in drain(sockets[label])] + self.assertEqual(actual, expected, f"after {' '.join(action[:2])}") + self.assertEqual(len(server.peers), 1) + self.assertEqual(server.stats.packets_dropped, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-ts/CHANGELOG.md b/packages/cultnet-ts/CHANGELOG.md index eae823c4..a5b8bc4e 100644 --- a/packages/cultnet-ts/CHANGELOG.md +++ b/packages/cultnet-ts/CHANGELOG.md @@ -9,8 +9,12 @@ All notable changes to `cultnet-ts` are recorded here, following - `CultNetRudpSocketTransportServer`: one UDP socket serving many peers, each remote address and port with its own reliable session (`peerConnected`, `frame`, `peerDisconnected` events; `send(peer, ...)` answers on the peer a frame came from). A second client no longer replaces the first; Connects beyond `maxPeers` (default 256) are dropped, and `close()` resolves once every peer's goodbye has left the socket. The shape follows the Rust `CultNetRudpServerHub` and C# `CultNetRudpSocketTransportServer`, and the wire format is unchanged. The optional `peerTimeoutMs` (idle peers ended and told so; default off) exists only in TypeScript. `CultNetRudpSocketTransportConnection` in `server` mode still serves exactly one peer. +- The `cultnet.snapshot_request.v1`, `cultnet.database_subscribe.v1` and `cultnet.snapshot_response_raw.v1` carriers and the selection payload types (`CultNetSelection`, `CultNetRawDocumentHeader`, `CultNetEdge`). `parseCultNetMessage` accepts them (absent members arrive as absent keys; a page carries exactly one of `headers`/`documents`; an empty or blank `schemas`/`keys` entry throws `CultNetSelectionInvalidError` naming its field), and `encodeCultNetMessageForWire` writes the reference's nil-keyed, declaration-ordered map. Judged against bytes the C# reference wrote (`contracts/cultnet/interop/carrier-vectors.cs-written.json`) and the selection vectors C# and Rust wrote. The selection engine (evaluation, cursors, the declaration-dependent door checks) is not ported. + ### Breaking +- `CultNetErrorMessage` carries `routingHint`, `code` and `details` (null when absent), as the reference writes them and `cultnet.error.schema.json` requires; the interface previously dropped them. New `CultNetErrorCode`, `CultNetErrorDetails` and `CultNetShardRoutingHint` types. + - `CultMeshCdnChunkRef` carries `id` (slot 4), the reference's element id. `encodeManifest` writes five-slot chunk references and `packContent` sets the decimal offset as the id, as the reference does; a manifest written before ids reads `""` and is rewritten with the slot. Content vectors regenerated from the reference. - Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): the Idunn runtime authority reader calls `readSingleFileStore`, which 0.14.x does not export. An absent authority file is now a plain `Error` without a code. diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index c766793b..acbf5a6a 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -65,6 +65,7 @@ Current message families: - `cultnet.snapshot_request.v0` - `cultnet.snapshot_response.v0` - `cultnet.snapshot_response_raw.v0` +- `cultnet.snapshot_request.v1`, `cultnet.database_subscribe.v1`, `cultnet.snapshot_response_raw.v1` (typed-selection carriers; no selection engine yet) - `cultnet.sample.change_name.v0` - `cultnet.sample.chat.v0` diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index ee060a1b..f3bfb2e8 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -33,8 +33,25 @@ import verseCatalogRequestSchema from "../contracts/cultnet/cultmesh.verse-catal import verseCatalogResponseSchema from "../contracts/cultnet/cultmesh.verse-catalog-response.schema.json"; import sessionOpenSchema from "../contracts/cultnet/cultmesh.session-open.schema.json"; import sessionAcceptedSchema from "../contracts/cultnet/cultmesh.session-accepted.schema.json"; +import selectionSchema from "../contracts/cultnet/cultnet.selection.schema.json"; +import rawDocumentHeaderSchema from "../contracts/cultnet/cultnet.raw-document-header.schema.json"; +import snapshotRequestV1Schema from "../contracts/cultnet/cultnet.snapshot-request.v1.schema.json"; +import snapshotResponseRawV1Schema from "../contracts/cultnet/cultnet.snapshot-response-raw.v1.schema.json"; +import databaseSubscribeV1Schema from "../contracts/cultnet/cultnet.database-subscribe.v1.schema.json"; import ghostlightAgentStateSchema from "../contracts/cultnet/ghostlight.agent-state.schema.json"; +import { + encodeSelectionCarrierForWire, + normalizeSelectionCarrierNulls, + type CultNetSelectionCarrier, +} from "./selection-carriers"; +import { + validateSelectionShape, + type CultNetEdge, + type CultNetRawDocumentHeader, + type CultNetSelection, +} from "./selection"; + export type CultNetWireContract = "cultnet.schema.v0" | "gamecult.networking.v0"; export type CultNetSchemaVersion = @@ -53,6 +70,9 @@ export type CultNetSchemaVersion = | "cultnet.snapshot_response.v0" | "cultnet.snapshot_response_raw.v0" | "cultnet.database_subscribe.v0" + | "cultnet.snapshot_request.v1" + | "cultnet.snapshot_response_raw.v1" + | "cultnet.database_subscribe.v1" | "cultnet.database_unsubscribe.v0" | "cultnet.database_change_raw.v0" | "cultnet.schema_catalog_request.v0" @@ -277,6 +297,46 @@ export interface CultNetDatabaseSubscribeMessage { supportedBodyTransports?: string[]; } +/** A document snapshot through a typed selection (docs/cultnet-selection-cut.md). */ +export interface CultNetSnapshotRequestV1Message { + schemaVersion: "cultnet.snapshot_request.v1"; + messageId: string; + selection: CultNetSelection; + shardId?: string; + shardEpoch?: number; +} + +/** A live subscription through a typed selection. */ +export interface CultNetDatabaseSubscribeV1Message { + schemaVersion: "cultnet.database_subscribe.v1"; + messageId: string; + subscriptionId: string; + selection: CultNetSelection; + /** Default true. */ + includeSnapshot?: boolean; + consumerRuntimeId?: string; + bodyIds?: string[]; + supportedBodyTransports?: string[]; +} + +/** One selection page: exactly one of `headers` / `documents` is present, by projection. */ +export interface CultNetSnapshotResponseRawV1Message { + schemaVersion: "cultnet.snapshot_response_raw.v1"; + messageId: string; + /** Rows on this page. */ + matched: number; + /** The snapshot this page is exact as of. */ + asOf: number; + /** Cursor for the next page, absent on the last page. */ + next?: string; + headers?: CultNetRawDocumentHeader[]; + documents?: CultNetRawDocumentRecord[]; + edges?: CultNetEdge[]; + shardId?: string; + shardEpoch?: number; + shardLogSequence?: number; +} + export interface CultNetDatabaseUnsubscribeMessage { schemaVersion: "cultnet.database_unsubscribe.v0"; messageId: string; @@ -689,6 +749,9 @@ export type CultNetMessage = | CultNetSnapshotResponseMessage | CultNetSnapshotResponseRawMessage | CultNetDatabaseSubscribeMessage + | CultNetSnapshotRequestV1Message + | CultNetSnapshotResponseRawV1Message + | CultNetDatabaseSubscribeV1Message | CultNetDatabaseUnsubscribeMessage | CultNetDatabaseChangeRawMessage | CultNetSchemaCatalogRequestMessage @@ -723,6 +786,9 @@ const CULTNET_MESSAGE_SCHEMAS = [ snapshotResponseSchema, snapshotResponseRawSchema, databaseSubscribeSchema, + snapshotRequestV1Schema, + snapshotResponseRawV1Schema, + databaseSubscribeV1Schema, databaseUnsubscribeSchema, databaseChangeRawSchema, schemaCatalogRequestSchema, @@ -741,6 +807,8 @@ const cultNetValidators = new Map(); for (const schema of [ documentRecordSchema, rawDocumentRecordSchema, + rawDocumentHeaderSchema, + selectionSchema, documentMutationContractSchema, transportProfileSchema, schemaDescriptorSchema, @@ -787,6 +855,20 @@ export function parseCultNetMessage(input: unknown, wireContract: CultNetWireCon return input as CultNetSnapshotResponseRawMessage; } + if (schemaVersion === "cultnet.snapshot_response_raw.v1") { + validateSnapshotResponseRawV1Message(input); + return input as CultNetSnapshotResponseRawV1Message; + } + + if (schemaVersion === "cultnet.snapshot_request.v1" || schemaVersion === "cultnet.database_subscribe.v1") { + // The door first, so an empty or blank schemas/keys entry is a typed refusal naming its field + // rather than a generic schema-validation failure. + const selection = (input as { selection?: unknown }).selection; + if (selection && typeof selection === "object") { + validateSelectionShape(selection as CultNetSelection); + } + } + if (schemaVersion === "cultnet.database_change_raw.v0") { validateDatabaseChangeRawMessage(input); return input as CultNetDatabaseChangeRawMessage; @@ -808,7 +890,14 @@ export function encodeCultNetMessageForWire( return encodeGameCultNetworkingMessage(message); } - return message; + switch (message.schemaVersion) { + case "cultnet.snapshot_request.v1": + case "cultnet.database_subscribe.v1": + case "cultnet.snapshot_response_raw.v1": + return encodeSelectionCarrierForWire(message as CultNetSelectionCarrier); + default: + return message; + } } export function validateGhostlightAgentState(input: unknown): GhostlightAgentStateDocument { @@ -932,6 +1021,7 @@ function encodeGameCultNetworkingMessage(message: CultNetMessage): [number, unkn decodeBase64Url(message.session, "LoginSuccessMessage.Session"), ]]; case "cultnet.error.v0": + // The gamecult.networking.v0 union's ErrorMessage carries only the text. return [4, [message.error]]; case "cultnet.sample.change_name.v0": return [5, [message.name]]; @@ -1136,6 +1226,89 @@ function validateSnapshotResponseRawMessage( }); } +function validateSnapshotResponseRawV1Message( + input: unknown, +): asserts input is CultNetSnapshotResponseRawV1Message { + const candidate = validateBinaryMessageRoot(input, "cultnet.snapshot_response_raw.v1"); + const fail = (path: string, problem: string): never => { + throw new Error(`Validation failed for cultnet.snapshot_response_raw.v1: ${path}: ${problem}`); + }; + for (const counter of ["matched", "asOf"] as const) { + const value = candidate[counter]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + fail(`/${counter}`, "must be a non-negative integer"); + } + } + if (typeof candidate.next !== "undefined" && typeof candidate.next !== "string") { + fail("/next", "must be a string"); + } + const hasHeaders = typeof candidate.headers !== "undefined"; + const hasDocuments = typeof candidate.documents !== "undefined"; + if (hasHeaders === hasDocuments) { + fail("/", "exactly one of headers and documents must be present"); + } + if (hasHeaders) { + if (!Array.isArray(candidate.headers)) fail("/headers", "must be an array"); + (candidate.headers as unknown[]).forEach((header, index) => validateRawDocumentHeader(header, `/headers/${index}`)); + } else { + if (!Array.isArray(candidate.documents)) fail("/documents", "must be an array"); + (candidate.documents as unknown[]).forEach((document, index) => validateRawDocumentRecord(document, `/documents/${index}`)); + } + if (typeof candidate.edges !== "undefined") { + if (!Array.isArray(candidate.edges)) fail("/edges", "must be an array"); + (candidate.edges as unknown[]).forEach((edge, index) => validateEdge(edge, `/edges/${index}`)); + } + requireOptionalNonEmptyString(candidate.shardId, "/shardId"); + for (const counter of ["shardEpoch", "shardLogSequence"] as const) { + const value = candidate[counter]; + if (typeof value !== "undefined" && (typeof value !== "number" || !Number.isSafeInteger(value))) { + fail(`/${counter}`, "must be an integer"); + } + } +} + +function validateRawDocumentHeader(input: unknown, instancePath: string): void { + if (!input || typeof input !== "object") { + throw new Error(`Validation failed for raw CultNet document header: ${instancePath}: must be an object`); + } + const candidate = input as Record; + requireNonEmptyString(candidate.schemaId, `${instancePath}/schemaId`); + requireOptionalNonEmptyString(candidate.schemaName, `${instancePath}/schemaName`); + requireOptionalNonEmptyString(candidate.schemaVersion, `${instancePath}/schemaVersion`); + requireOptionalNonEmptyString(candidate.schemaContentHash, `${instancePath}/schemaContentHash`); + requireNonEmptyString(candidate.recordKey, `${instancePath}/recordKey`); + requireNonEmptyString(candidate.storedAt, `${instancePath}/storedAt`); + requireOptionalNonEmptyString(candidate.sourceRuntimeId, `${instancePath}/sourceRuntimeId`); + requireOptionalNonEmptyString(candidate.sourceAgentId, `${instancePath}/sourceAgentId`); + requireOptionalNonEmptyString(candidate.sourceRole, `${instancePath}/sourceRole`); + requireOptionalStringArray(candidate.tags, `${instancePath}/tags`); +} + +function validateEdge(input: unknown, instancePath: string): void { + const fail = (path: string, problem: string): never => { + throw new Error(`Validation failed for cultnet.snapshot_response_raw.v1: ${path}: ${problem}`); + }; + if (!input || typeof input !== "object") { + fail(instancePath, "must be an object"); + } + const candidate = input as Record; + for (const end of ["from", "to"] as const) { + const ref = candidate[end] as Record | undefined; + if (!ref || typeof ref !== "object") { + fail(`${instancePath}/${end}`, "must be an object"); + } + requireNonEmptyString(ref!.schemaId, `${instancePath}/${end}/schemaId`); + requireNonEmptyString(ref!.recordKey, `${instancePath}/${end}/recordKey`); + } + requireNonEmptyString(candidate.role, `${instancePath}/role`); + if (typeof candidate.payloadEncoding !== "undefined" && candidate.payloadEncoding !== "messagepack") { + fail(`${instancePath}/payloadEncoding`, "must equal messagepack"); + } + if (typeof candidate.payload !== "undefined" && !(candidate.payload instanceof Uint8Array)) { + fail(`${instancePath}/payload`, "must be a Uint8Array"); + } +} + function validateDatabaseChangeRawMessage( input: unknown, ): asserts input is CultNetDatabaseChangeRawMessage { @@ -1168,6 +1341,7 @@ function validateBinaryMessageRoot( schemaVersion: | "cultnet.document_put_raw.v0" | "cultnet.snapshot_response_raw.v0" + | "cultnet.snapshot_response_raw.v1" | "cultnet.database_change_raw.v0", ): Record { if (!input || typeof input !== "object") { @@ -1270,6 +1444,11 @@ function normalizeCultNetOptionalNulls( "supportedBodyTransports", ]); return; + case "cultnet.snapshot_request.v1": + case "cultnet.database_subscribe.v1": + case "cultnet.snapshot_response_raw.v1": + normalizeSelectionCarrierNulls(schemaVersion, candidate); + return; case "cultnet.operation_request.v0": stripNullProperties(candidate, ["payloadEncoding", "sourceRuntimeId", "targetRuntimeId"]); return; @@ -1339,6 +1518,11 @@ export const cultNetSchemas = { snapshotResponseSchema, snapshotResponseRawSchema, databaseSubscribeSchema, + selectionSchema, + rawDocumentHeaderSchema, + snapshotRequestV1Schema, + snapshotResponseRawV1Schema, + databaseSubscribeV1Schema, databaseUnsubscribeSchema, databaseChangeRawSchema, schemaDescriptorSchema, diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index af53a437..5e654c28 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -3,6 +3,9 @@ export { encodeCultNetMessageForWire, parseCultNetMessage, validateGhostlightAgentState, + type CultNetDatabaseSubscribeV1Message, + type CultNetSnapshotRequestV1Message, + type CultNetSnapshotResponseRawV1Message, type CultNetDocumentDeleteMessage, type CultNetDocumentMutationContract, type CultNetDocumentPutMessage, @@ -66,6 +69,19 @@ export { type GhostlightVariableMap, type GhostlightWorldState, } from "./contracts"; +export { + CultNetSelectionInvalidError, + validateSelectionShape, + type CultNetCitation, + type CultNetEdge, + type CultNetFieldPredicate, + type CultNetIncoming, + type CultNetRawDocumentHeader, + type CultNetRecordRef, + type CultNetSelection, + type CultNetSelectionOperator, + type CultNetSelectionProjection, +} from "./selection"; export { base64ToBytes, bytesToBase64, diff --git a/packages/cultnet-ts/src/selection-carriers.ts b/packages/cultnet-ts/src/selection-carriers.ts new file mode 100644 index 00000000..21090f03 --- /dev/null +++ b/packages/cultnet-ts/src/selection-carriers.ts @@ -0,0 +1,178 @@ +// The two shape conversions the selection/v1 carriers need between the reference's wire and TypeScript: +// the reference writes every declared key, nil included, in declaration order (MessagePack-CSharp map +// mode; schema notes R-S/R-X), so a received message arrives with explicit nulls that are normalised +// to absent keys, and a sent message is rebuilt with every key in the reference's order. + +import type { + CultNetDatabaseSubscribeV1Message, + CultNetRawDocumentRecord, + CultNetSnapshotRequestV1Message, + CultNetSnapshotResponseRawV1Message, +} from "./contracts"; +import type { CultNetEdge, CultNetRawDocumentHeader, CultNetRecordRef, CultNetSelection } from "./selection"; + +export type CultNetSelectionCarrier = + | CultNetSnapshotRequestV1Message + | CultNetDatabaseSubscribeV1Message + | CultNetSnapshotResponseRawV1Message; + +type Bag = Record; + +const isBag = (value: unknown): value is Bag => !!value && typeof value === "object" && !Array.isArray(value); + +function stripNulls(candidate: unknown, keys: readonly string[]): void { + if (!isBag(candidate)) return; + for (const key of keys) { + if (candidate[key] === null) delete candidate[key]; + } +} + +function eachBag(list: unknown, visit: (item: Bag) => void): void { + if (!Array.isArray(list)) return; + for (const item of list) { + if (isBag(item)) visit(item); + } +} + +/** Drops the nil an absent selection member travels as, at every depth the selection carries one. */ +export function normalizeSelectionNulls(selection: unknown): void { + stripNulls(selection, ["schemas", "keys", "fields", "cites", "cited", "limit", "cursor"]); + if (!isBag(selection)) return; + eachBag(selection.fields, field => stripNulls(field, ["values", "number"])); + stripNulls(selection.cites, ["role"]); +} + +/** Drops the nil an absent member travels as, for one selection/v1 carrier message. */ +export function normalizeSelectionCarrierNulls( + schemaVersion: CultNetSelectionCarrier["schemaVersion"], + candidate: Bag, +): void { + switch (schemaVersion) { + case "cultnet.snapshot_request.v1": + stripNulls(candidate, ["shardId", "shardEpoch"]); + normalizeSelectionNulls(candidate.selection); + return; + case "cultnet.database_subscribe.v1": + stripNulls(candidate, ["consumerRuntimeId", "bodyIds", "supportedBodyTransports"]); + normalizeSelectionNulls(candidate.selection); + return; + case "cultnet.snapshot_response_raw.v1": + stripNulls(candidate, ["next", "headers", "documents", "edges", "shardId", "shardEpoch", "shardLogSequence"]); + eachBag(candidate.headers, header => stripNulls(header, HEADER_OPTIONAL)); + eachBag(candidate.documents, document => stripNulls(document, HEADER_OPTIONAL)); + eachBag(candidate.edges, edge => stripNulls(edge, ["payloadEncoding", "payload"])); + return; + } +} + +const HEADER_OPTIONAL = ["schemaName", "schemaVersion", "schemaContentHash", "sourceRuntimeId", "sourceAgentId", "sourceRole", "tags"] as const; + +const orNull = (value: T | undefined | null): T | null => value ?? null; + +function recordRefWire(ref: CultNetRecordRef) { + return { schemaId: ref.schemaId, recordKey: ref.recordKey }; +} + +function selectionWire(selection: CultNetSelection) { + return { + schemas: orNull(selection.schemas), + keys: orNull(selection.keys), + fields: selection.fields + ? selection.fields.map(field => ({ + index: field.index, + op: field.op, + values: orNull(field.values), + number: orNull(field.number), + })) + : null, + cites: selection.cites + ? { target: recordRefWire(selection.cites.target), role: orNull(selection.cites.role) } + : null, + cited: selection.cited ? { role: selection.cited.role, exists: selection.cited.exists } : null, + projection: selection.projection ?? "header", + descending: selection.descending ?? false, + limit: orNull(selection.limit), + cursor: orNull(selection.cursor), + }; +} + +function headerWire(header: CultNetRawDocumentHeader) { + return { + schemaId: header.schemaId, + schemaName: orNull(header.schemaName), + schemaVersion: orNull(header.schemaVersion), + schemaContentHash: orNull(header.schemaContentHash), + recordKey: header.recordKey, + storedAt: header.storedAt, + sourceRuntimeId: orNull(header.sourceRuntimeId), + sourceAgentId: orNull(header.sourceAgentId), + sourceRole: orNull(header.sourceRole), + tags: orNull(header.tags), + }; +} + +function documentWire(document: CultNetRawDocumentRecord) { + return { + schemaId: document.schemaId, + schemaName: orNull(document.schemaName), + schemaVersion: orNull(document.schemaVersion), + schemaContentHash: orNull(document.schemaContentHash), + recordKey: document.recordKey, + storedAt: document.storedAt, + payloadEncoding: document.payloadEncoding, + payload: document.payload, + sourceRuntimeId: orNull(document.sourceRuntimeId), + sourceAgentId: orNull(document.sourceAgentId), + sourceRole: orNull(document.sourceRole), + tags: orNull(document.tags), + }; +} + +function edgeWire(edge: CultNetEdge) { + return { + from: recordRefWire(edge.from), + role: edge.role, + to: recordRefWire(edge.to), + payloadEncoding: orNull(edge.payloadEncoding), + payload: orNull(edge.payload), + }; +} + +/** The reference's nil-keyed, declaration-ordered shape for one selection/v1 carrier message. */ +export function encodeSelectionCarrierForWire(message: CultNetSelectionCarrier): unknown { + switch (message.schemaVersion) { + case "cultnet.snapshot_request.v1": + return { + schemaVersion: message.schemaVersion, + messageId: message.messageId, + selection: selectionWire(message.selection), + shardId: orNull(message.shardId), + shardEpoch: orNull(message.shardEpoch), + }; + case "cultnet.database_subscribe.v1": + return { + schemaVersion: message.schemaVersion, + messageId: message.messageId, + subscriptionId: message.subscriptionId, + selection: selectionWire(message.selection), + includeSnapshot: message.includeSnapshot ?? true, + consumerRuntimeId: orNull(message.consumerRuntimeId), + bodyIds: orNull(message.bodyIds), + supportedBodyTransports: orNull(message.supportedBodyTransports), + }; + case "cultnet.snapshot_response_raw.v1": + return { + schemaVersion: message.schemaVersion, + messageId: message.messageId, + matched: message.matched, + asOf: message.asOf, + next: orNull(message.next), + headers: message.headers ? message.headers.map(headerWire) : null, + documents: message.documents ? message.documents.map(documentWire) : null, + edges: message.edges ? message.edges.map(edgeWire) : null, + shardId: orNull(message.shardId), + shardEpoch: orNull(message.shardEpoch), + shardLogSequence: orNull(message.shardLogSequence), + }; + } +} diff --git a/packages/cultnet-ts/src/selection.ts b/packages/cultnet-ts/src/selection.ts new file mode 100644 index 00000000..2bba1ebe --- /dev/null +++ b/packages/cultnet-ts/src/selection.ts @@ -0,0 +1,115 @@ +// cultnet.selection.v1 payload shapes (docs/cultnet-selection-cut.md, section 2), the carriers' common +// vocabulary: one typed selection, the header/edge rows a page answers with, and the door refusal. +// The messages that carry them live in contracts.ts; the nil-keyed wire encoding, null normalization and +// response validation live in selection-carriers.ts. + +export type CultNetSelectionOperator = "any_of" | "lt" | "le" | "ge" | "gt"; +export type CultNetSelectionProjection = "header" | "document"; + +/** One predicate over a single declared index alias. `values` iff any_of; `number` (canonical decimal string) iff a comparison. */ +export interface CultNetFieldPredicate { + index: string; + op: CultNetSelectionOperator; + values?: string[]; + number?: string; +} + +/** Identifies one row on the wire by schema and record key. */ +export interface CultNetRecordRef { + schemaId: string; + recordKey: string; +} + +/** Selects rows whose declared reference names `target`; an absent `role` matches any declared reference. */ +export interface CultNetCitation { + target: CultNetRecordRef; + role?: string; +} + +/** Selects rows some other row does (`exists`) or does not name in `role`. */ +export interface CultNetIncoming { + role: string; + exists: boolean; +} + +/** Absent `schemas`/`keys` mean every schema/key; a present empty list is refused at the door. */ +export interface CultNetSelection { + schemas?: string[]; + keys?: string[]; + fields?: CultNetFieldPredicate[]; + cites?: CultNetCitation; + cited?: CultNetIncoming; + /** Default "header". */ + projection?: CultNetSelectionProjection; + descending?: boolean; + /** Clamped to 1..=200 by the answering server. */ + limit?: number; + /** Opaque; minted by the answering server. */ + cursor?: string; +} + +/** A raw document record minus its payload and payloadEncoding. */ +export interface CultNetRawDocumentHeader { + schemaId: string; + schemaName?: string; + schemaVersion?: string; + schemaContentHash?: string; + recordKey: string; + storedAt: string; + sourceRuntimeId?: string; + sourceAgentId?: string; + sourceRole?: string; + tags?: string[]; +} + +/** One edge a hop traversed. `payload` is present under document projection only. */ +export interface CultNetEdge { + from: CultNetRecordRef; + role: string; + to: CultNetRecordRef; + payloadEncoding?: "messagepack"; + payload?: Uint8Array; +} + +/** + * Refused typed at the door: a selection the declared shape cannot support. Carried back as + * `cultnet.error.v0 { code: "selection_invalid", details: { field, value } }`. + */ +export class CultNetSelectionInvalidError extends Error { + constructor( + readonly field: string, + readonly value: string | null, + message: string, + ) { + super(message); + this.name = "CultNetSelectionInvalidError"; + } +} + +/** + * The declaration-independent half of the door, matching the reference's `ValidateShape`: an empty or + * blank `schemas`/`keys` entry or list, and an unrecognised `projection`. Checks that need the + * declared schemas (index aliases, reference roles, cites target) belong to the selection engine. + */ +export function validateSelectionShape(selection: CultNetSelection): void { + const list = (field: "schemas" | "keys") => { + const entries = selection[field]; + if (entries === undefined || entries === null) return; + if (entries.length === 0) { + throw new CultNetSelectionInvalidError( + field, null, `selection.${field} is present and empty; omit it to reach every ${field === "schemas" ? "schema" : "key"}.`); + } + for (const entry of entries) { + if (typeof entry !== "string" || entry.trim().length === 0) { + throw new CultNetSelectionInvalidError(field, entry, `selection.${field} carries an empty or whitespace entry.`); + } + } + }; + list("schemas"); + list("keys"); + const projection = selection.projection ?? "header"; + if (projection !== "header" && projection !== "document") { + throw new CultNetSelectionInvalidError( + "projection", String(projection), `selection.projection "${String(projection)}" is neither "header" nor "document".`); + } +} diff --git a/packages/cultnet-ts/test/carrier-vectors.test.ts b/packages/cultnet-ts/test/carrier-vectors.test.ts new file mode 100644 index 00000000..63f5826f --- /dev/null +++ b/packages/cultnet-ts/test/carrier-vectors.test.ts @@ -0,0 +1,171 @@ +// The reference's own bytes for the cultnet.error.v0 code/details shape and the selection/v1 +// carriers (`contracts/cultnet/interop/carrier-vectors.cs-written.json`, written by the C# +// CarrierVectorTests). Each vector is decoded, parsed, and re-encoded: the re-encoding must be the +// reference's exact bytes, and the decoded message must equal the reference's own JSON of the object. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { decode, encode } from "@msgpack/msgpack"; + +import { encodeCultNetMessageForWire, parseCultNetMessage } from "../src/contracts"; + +const FILE = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "carrier-vectors.cs-written.json"); +type Vector = { name: string; schemaVersion: string; messagePackBase64: string; expected: unknown }; +const vectors: Vector[] = JSON.parse(readFileSync(FILE, "utf8")).vectors; +const vectorsFor = (prefix: string) => vectors.filter(v => v.name.startsWith(prefix)); +const bytesOf = (v: Vector) => new Uint8Array(Buffer.from(v.messagePackBase64, "base64")); + +/** Uint8Array to base64, recursively, so a decoded message compares with the reference's JSON. */ +function jsonShape(value: unknown): unknown { + if (value instanceof Uint8Array) return Buffer.from(value).toString("base64"); + if (Array.isArray(value)) return value.map(jsonShape); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, jsonShape(v)])); + } + return value; +} + +const reencode = (message: Parameters[0]) => + new Uint8Array(encode(encodeCultNetMessageForWire(message), { ignoreUndefined: true })); + +test("every error vector keeps the reference's code and details through parse and re-encode", () => { + const errors = vectorsFor("error_"); + assert.ok(errors.length >= 5); + for (const v of errors) { + const raw = decode(bytesOf(v)); + assert.deepEqual(jsonShape(raw), v.expected, `${v.name}: decoded bytes`); + const message = parseCultNetMessage(raw); + assert.equal(message.schemaVersion, "cultnet.error.v0"); + assert.deepEqual(jsonShape(message), v.expected, `${v.name}: parsed message`); + assert.deepEqual(reencode(message), bytesOf(v), `${v.name}: re-encoded bytes`); + } +}); + +test("an error whose code is outside the schema's closed enum is refused", () => { + const raw = decode(bytesOf(vectors.find(v => v.name === "error_cursor_stale")!)) as Record; + assert.throws(() => parseCultNetMessage({ ...raw, code: "made_up_code" }), /cultnet\.error\.v0/); +}); + +// ---- selection/v1 carriers ---- + +for (const prefix of ["snapshot_request_v1", "database_subscribe_v1", "snapshot_response_raw_v1"]) { + test(`${prefix} vectors: decode, parse, and re-encode to the reference's exact bytes`, () => { + const group = vectorsFor(prefix); + assert.ok(group.length >= 2, `${prefix}: vectors present`); + for (const v of group) { + const raw = decode(bytesOf(v)); + assert.deepEqual(jsonShape(raw), v.expected, `${v.name}: decoded bytes`); + const message = parseCultNetMessage(raw); + assert.equal(message.schemaVersion, v.schemaVersion); + assert.deepEqual(reencode(message), bytesOf(v), `${v.name}: re-encoded bytes`); + } + }); +} + +test("a parsed v1 message drops the nil an absent member travels as", () => { + const full = parseCultNetMessage(decode(bytesOf(vectors.find(v => v.name === "snapshot_request_v1_defaults")!))) as unknown as + { selection: Record; shardId?: unknown }; + assert.deepEqual(Object.keys(full.selection).sort(), ["descending", "projection"]); + assert.equal("shardId" in full, false); + const page = parseCultNetMessage(decode(bytesOf(vectors.find(v => v.name === "snapshot_response_raw_v1_headers")!))) as unknown as + { headers: Record[]; documents?: unknown }; + assert.equal("documents" in page, false); + assert.equal("tags" in page.headers[0], false); + assert.deepEqual(page.headers[1].tags, ["alpha", "beta"]); +}); + +test("edge payloads and document payloads arrive as bytes, not text", () => { + const page = parseCultNetMessage(decode(bytesOf(vectors.find(v => v.name === "snapshot_response_raw_v1_documents_and_edges")!))) as unknown as + { edges: { payload?: Uint8Array }[]; documents: { payload: Uint8Array }[] }; + assert.deepEqual(Array.from(page.edges[1].payload!), [1, 2, 3]); + assert.equal("payload" in page.edges[0], false); + assert.deepEqual(Array.from(page.documents[0].payload), [0x81, 0xa1, 0x6b, 0x01]); +}); + +const request = () => decode(bytesOf(vectors.find(v => v.name === "snapshot_request_v1_full_selection_and_shard")!)) as + { selection: Record } & Record; +const refusedField = (message: unknown): string | undefined => { + try { + parseCultNetMessage(message); + } catch (error) { + return (error as { field?: string }).field; + } + return "accepted"; +}; + +test("the door refuses an empty or blank schemas/keys entry with a typed field, never as every-schema", () => { + const withSelection = (patch: Record) => ({ ...request(), selection: { ...request().selection, ...patch } }); + assert.equal(refusedField(withSelection({ schemas: [] })), "schemas"); + assert.equal(refusedField(withSelection({ keys: ["a-eq", " "] })), "keys"); + assert.equal(refusedField(withSelection({ projection: "everything" })), "projection"); +}); + +test("the schema refuses a comparison whose number is not a canonical decimal, and a predicate that mixes values and number", () => { + const withField = (field: Record) => { + const message = request(); + return { ...message, selection: { ...message.selection, fields: [field] } }; + }; + for (const number of ["05", "1.50", "+1", "1e3", "-0", " 1"]) { + assert.throws(() => parseCultNetMessage(withField({ index: "mass", op: "ge", values: null, number })), /snapshot_request\.v1/, number); + } + assert.throws(() => parseCultNetMessage(withField({ index: "mass", op: "ge", values: ["1"], number: "1" })), /snapshot_request\.v1/); + assert.throws(() => parseCultNetMessage(withField({ index: "mass", op: "near", values: null, number: "1" })), /snapshot_request\.v1/); +}); + +test("a selection page with both or neither of headers and documents is refused", () => { + const page = decode(bytesOf(vectors.find(v => v.name === "snapshot_response_raw_v1_headers")!)) as Record; + assert.throws(() => parseCultNetMessage({ ...page, documents: [] }), /exactly one of headers and documents/); + assert.throws(() => parseCultNetMessage({ ...page, headers: null }), /exactly one of headers and documents/); +}); + +// ---- selection vectors: the reference's and Rust's selection bytes ---- + +const INTEROP = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop"); +type SelectionVector = { name: string; selectionMessagePackBase64: string; expectedRefusalField?: string | null }; +const selectionVectors = (file: string): SelectionVector[] => + JSON.parse(readFileSync(join(INTEROP, file), "utf8")).vectors; + +import { CultNetSelectionInvalidError, validateSelectionShape, type CultNetSelection } from "../src/selection"; + +for (const file of ["selection-vectors.cs-written.json", "selection-vectors.rs-written.json"]) { + test(`${file}: every selection decodes inside a v1 request and the schema door agrees with the vector`, () => { + const all = selectionVectors(file); + assert.ok(all.length >= 20); + for (const v of all) { + const selection = decode(new Uint8Array(Buffer.from(v.selectionMessagePackBase64, "base64"))); + const message = { schemaVersion: "cultnet.snapshot_request.v1", messageId: "m", selection, shardId: null, shardEpoch: null }; + if (v.expectedRefusalField === "schemas" || v.expectedRefusalField === "keys") { + assert.equal(refusedField(message), v.expectedRefusalField, `${v.name}: shape refusal`); + continue; + } + if (v.expectedRefusalField) { + // Declaration-dependent refusals (cites target, index aliases) need the selection engine's schemas. + const parsed = parseCultNetMessage(message) as unknown as { selection: CultNetSelection }; + validateSelectionShape(parsed.selection); + continue; + } + const parsed = parseCultNetMessage(message) as unknown as { selection: CultNetSelection }; + validateSelectionShape(parsed.selection); + const rewritten = new Uint8Array(encode( + (encodeCultNetMessageForWire(parsed as never) as { selection: unknown }).selection)); + if (file.includes("cs-written")) { + // The reference's bytes are the nil-keyed shape: our encoder must reproduce them exactly. + assert.deepEqual(rewritten, new Uint8Array(Buffer.from(v.selectionMessagePackBase64, "base64")), + `${v.name}: selection re-encodes to the reference's bytes`); + } else { + // Rust omits absent members; what it means must survive our nil-keyed rewrite unchanged. + const again = parseCultNetMessage({ ...message, selection: decode(rewritten) }) as unknown as { selection: CultNetSelection }; + assert.deepEqual(again.selection, parsed.selection, `${v.name}: selection meaning survives re-encode`); + } + } + }); +} + +test("CultNetSelectionInvalidError carries the field and offending value", () => { + assert.throws( + () => validateSelectionShape({ keys: ["ok", ""] }), + (error: unknown) => error instanceof CultNetSelectionInvalidError && error.field === "keys" && error.value === "", + ); +}); diff --git a/tests/GameCult.Networking.Tests/CarrierVectorTests.cs b/tests/GameCult.Networking.Tests/CarrierVectorTests.cs new file mode 100644 index 00000000..8d50bb76 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CarrierVectorTests.cs @@ -0,0 +1,178 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // The C# reference's own bytes for the selection/v1 message carriers and the cultnet.error.v0 + // code/details shape, written to contracts/cultnet/interop/carrier-vectors.cs-written.json for + // the runtimes that decode and re-encode them (packages/cultnet-ts reads it; a Rust or Python + // reader may follow). Each vector carries the whole message's MessagePack bytes plus `expected`, + // the reference's own camelCase JSON of the same object (byte[] as base64), so a reader judges + // against what the reference serialized rather than a second hand-written spec. + // + // Gated like SelectionParityVectorTests: CULTNET_WRITE_VECTORS=1 rewrites the committed file; + // otherwise the committed bytes are compared with a fresh serialization so reference drift fails + // here instead of in a reader. + public sealed class CarrierVectorTests + { + private const string LeafA = "sha256:ff35f3d9d6de3f1066b9369e73fe6c7648e88368d18efc2275c994730cd84cff"; + private const string CiterId = "sha256:9d0bc9aebc3619b07aabf564f98667afde41444c1b866661d777687d443608dc"; + + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + private static CultNetSelection FullSelection() => new() + { + Schemas = new[] { "leaf_a.v9", LeafA }, + Keys = new[] { "a-eq", "\U0001F600-row" }, + Fields = new[] + { + new CultNetFieldPredicate { Index = "kind", Op = "any_of", Values = new[] { "weapon", "tool" } }, + new CultNetFieldPredicate { Index = "mass", Op = "ge", Number = "-2233759.25" } + }, + Cites = new CultNetCitation { Target = new CultNetRecordRef { SchemaId = LeafA, RecordKey = "a-eq" }, Role = "Design" }, + Cited = new CultNetIncoming { Role = "Design", Exists = false }, + Projection = CultNetSelectionProjections.Document, + Descending = true, + Limit = 200, + Cursor = "opaque-cursor" + }; + + private static CultNetRawDocumentHeader Header(string key, params string[] tags) => new() + { + SchemaId = LeafA, + SchemaName = "leaf_a", + SchemaVersion = "1", + SchemaContentHash = "abc", + RecordKey = key, + StoredAt = "1970-01-01T00:00:00.000Z", + SourceRuntimeId = "rt", + SourceAgentId = "agent", + SourceRole = "role", + Tags = tags.Length == 0 ? null : tags + }; + + private static IEnumerable<(string Name, ICultNetSchemaMessage Message)> Cases() + { + yield return ("snapshot_request_v1_defaults", new CultNetSnapshotRequestV1Message { MessageId = "m" }); + yield return ("snapshot_request_v1_full_selection_and_shard", new CultNetSnapshotRequestV1Message + { + MessageId = "m-full", Selection = FullSelection(), ShardId = "shard-1", ShardEpoch = 7 + }); + yield return ("database_subscribe_v1_defaults", new CultNetDatabaseSubscribeV1Message { MessageId = "m", SubscriptionId = "sub" }); + yield return ("database_subscribe_v1_full", new CultNetDatabaseSubscribeV1Message + { + MessageId = "m-full", SubscriptionId = "sub-full", Selection = FullSelection(), IncludeSnapshot = false, + ConsumerRuntimeId = "consumer", BodyIds = new[] { "body-a" }, SupportedBodyTransports = new[] { "rudp", "quic" } + }); + yield return ("snapshot_response_raw_v1_headers", new CultNetSnapshotResponseRawV1Message + { + MessageId = "m", Matched = 2, AsOf = 5, Next = "next-cursor", + Headers = new[] { Header("a-eq"), Header("a-lo", "alpha", "beta") }, + ShardId = "shard-1", ShardEpoch = 3, ShardLogSequence = 41 + }); + yield return ("snapshot_response_raw_v1_documents_and_edges", new CultNetSnapshotResponseRawV1Message + { + MessageId = "m", Matched = 1, AsOf = 5, + Documents = new[] + { + new CultNetRawDocumentRecord { SchemaId = LeafA, RecordKey = "a-eq", StoredAt = "1970-01-01T00:00:00.000Z", Payload = new byte[] { 0x81, 0xA1, 0x6B, 0x01 } } + }, + Edges = new[] + { + new CultNetEdge + { + From = new CultNetRecordRef { SchemaId = CiterId, RecordKey = "citer-1" }, Role = "Design", + To = new CultNetRecordRef { SchemaId = LeafA, RecordKey = "a-eq" } + }, + new CultNetEdge + { + From = new CultNetRecordRef { SchemaId = CiterId, RecordKey = "citer-1" }, Role = "Design", + To = new CultNetRecordRef { SchemaId = LeafA, RecordKey = "a-eq" }, + PayloadEncoding = "messagepack", Payload = new byte[] { 1, 2, 3 } + } + } + }); + yield return ("error_plain", new CultNetErrorMessage { Error = "boom" }); + yield return ("error_selection_invalid", CultNetErrorMessage.ForSelectionInvalid( + new CultNetSelectionInvalidException("fields[0].index", "kind", "fields[0].index \"kind\" is not declared."))); + yield return ("error_cursor_stale", new CultNetErrorMessage + { + Error = "cursor_stale: stale", Code = "cursor_stale", Details = new CultNetErrorDetails { AsOf = 3, Current = 9 } + }); + yield return ("error_cursor_invalid_no_details", new CultNetErrorMessage { Error = "cursor_invalid: bad", Code = "cursor_invalid" }); + yield return ("error_write_not_authorized", new CultNetErrorMessage + { + Error = "write_not_authorized", Code = "write_not_authorized", Details = new CultNetErrorDetails { Field = "recordKey", Value = "k" } + }); + } + + private static object Vector(string name, ICultNetSchemaMessage message) + { + var bytes = MessagePackSerializer.Serialize(message.GetType(), message, CultNetSchemaMessageSerialization.Options); + return new + { + name, + schemaVersion = message.SchemaVersion, + messagePackBase64 = Convert.ToBase64String(bytes), + expected = WithoutDerivedMembers(JsonSerializer.SerializeToNode(message, message.GetType(), JsonOptions)) + }; + } + + // CultNetSelection.HasHop is an [IgnoreMember] convenience, not wire; System.Text.Json would emit it. + private static JsonNode? WithoutDerivedMembers(JsonNode? node) + { + switch (node) + { + case JsonObject obj: + obj.Remove("hasHop"); + foreach (var child in obj.Select(pair => pair.Value).ToArray()) WithoutDerivedMembers(child); + break; + case JsonArray array: + foreach (var child in array) WithoutDerivedMembers(child); + break; + } + return node; + } + + private static string VectorPath() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !File.Exists(Path.Combine(directory.FullName, "CultLib.sln"))) + directory = directory.Parent; + var root = directory?.FullName ?? throw new InvalidOperationException("CultLib.sln was not found above the test directory."); + return Path.Combine(root, "contracts", "cultnet", "interop", "carrier-vectors.cs-written.json"); + } + + [Test] + public void WriteCarrierVectors() + { + if (Environment.GetEnvironmentVariable("CULTNET_WRITE_VECTORS") != "1") + Assert.Ignore("Set CULTNET_WRITE_VECTORS=1 to (re)write the committed vector file."); + + var vectors = Cases().Select(c => Vector(c.Name, c.Message)).ToArray(); + File.WriteAllText(VectorPath(), JsonSerializer.Serialize(new { vectors }, new JsonSerializerOptions { WriteIndented = true })); + } + + [Test] + public void CommittedCarrierVectorsMatchAFreshSerialization() + { + using var committed = JsonDocument.Parse(File.ReadAllText(VectorPath())); + var byName = committed.RootElement.GetProperty("vectors").EnumerateArray() + .ToDictionary(v => v.GetProperty("name").GetString()!, v => v.GetProperty("messagePackBase64").GetString()!); + + Assert.That(byName.Count, Is.EqualTo(Cases().Count()), "the committed file must carry exactly the cases this writer produces"); + foreach (var (name, message) in Cases()) + { + var fresh = Convert.ToBase64String(MessagePackSerializer.Serialize(message.GetType(), message, CultNetSchemaMessageSerialization.Options)); + Assert.That(byName[name], Is.EqualTo(fresh), $"{name}: the committed bytes drifted from the reference serializer"); + } + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs new file mode 100644 index 00000000..0985d516 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs @@ -0,0 +1,155 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// The multi-peer listener's golden in tests/vectors/rudp-multipeer-server: two peers on one socket, the + /// datagrams each sends and the datagrams the C# reference answers with. The C# reference is the only writer + /// (CULTNET_WRITE_VECTORS=1); every other run replays the scenario and compares it byte for byte. Other runtimes' + /// listeners replay the same transcript and are held to the same answers. + /// + public class CultNetRudpMultiPeerVectorTests + { + private const string WriteVariable = "CULTNET_WRITE_VECTORS"; + private const uint ConnectionId = 0x43554c54; + + private sealed class Client + { + public string Label = ""; + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + } + + private static string Hex(byte[] bytes) => Convert.ToHexString(bytes).ToLowerInvariant(); + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + [Test] + public void TheReferenceListenerAnswersTwoPeersWithThePinnedBytes() + { + var transcript = Run(); + var path = Path.Combine(VectorDirectory(), "transcript.txt"); + if (Environment.GetEnvironmentVariable(WriteVariable) == "1") + { + File.WriteAllText(path, transcript); + Assert.Pass("wrote " + path); + } + Assert.That(transcript, Is.EqualTo(File.ReadAllText(path))); + } + + private static string VectorDirectory() + { + var directory = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (directory != null && !Directory.Exists(Path.Combine(directory.FullName, "tests", "vectors"))) + directory = directory.Parent; + if (directory == null) throw new InvalidOperationException("tests/vectors not found above the test directory."); + return Path.Combine(directory.FullName, "tests", "vectors", "rudp-multipeer-server"); + } + + private static string Run() + { + var log = new StringBuilder(); + log.Append("# config connection_id=0x43554c54 initial_sequence=500 max_fragment_bytes=8 accept_payload=616363\n"); + log.Append("# in | send | out ; peer labels name remote endpoints\n"); + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-vector", + Socket = serverSocket, + ConnectionId = ConnectionId, + InitialSequence = 500, + MaxFragmentBytes = 8, + AcceptPayload = Encoding.ASCII.GetBytes("acc") + }); + var target = serverSocket.LocalEndPoint!; + Client Make(string label, uint sequence) => new Client + { + Label = label, + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = sequence }) + }; + var x = Make("X", 10); + var y = Make("Y", 20); + var clients = new[] { x, y }; + + void Deliver(Client client, CultNetRudpPacket packet) + { + var wire = CultNetRudpPacketCodec.Encode(packet); + log.Append($"in {client.Label} {Hex(wire)}\n"); + client.Socket.SendTo(wire, target); + while (!server.TryReceiveOnce(out _)) + System.Threading.Thread.Sleep(1); + Flush(); + } + + void Flush() + { + foreach (var client in clients) + { + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (client.Socket.Poll(30_000, SelectMode.SelectRead)) + { + var received = client.Socket.ReceiveFrom(buffer, ref from); + var wire = buffer.AsSpan(0, received).ToArray(); + log.Append($"out {client.Label} {Hex(wire)}\n"); + // Clients that stay connected take the server's packets so their acks stay true. + var packet = CultNetRudpPacketCodec.Decode(wire); + if (client.Session.Connected || packet.PacketType == CultNetRudpPacketType.Accept) + client.Session.Receive(packet, 0); + } + } + } + + CultNetRudpSocketServerPeer PeerOf(Client client) => + server.Peers.Single(p => p.RemoteEndPoint.Equals(client.Socket.LocalEndPoint)); + + void Send(Client client, string channel, byte[] payload) + { + log.Append($"send {client.Label} {channel} {Hex(payload)}\n"); + server.Send(PeerOf(client), channel, payload); + Flush(); + } + + CultNetRudpPacket Data(Client client, string text) => + client.Session.Send("schema", Encoding.ASCII.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + + var xConnect = x.Session.CreateConnect(0, Encoding.ASCII.GetBytes("hx")); + Deliver(x, xConnect); + Deliver(y, y.Session.CreateConnect(0, Encoding.ASCII.GetBytes("hy"))); + // A repeat of X's accepted Connect is answered with the Accept owed, not a new session. + Deliver(x, xConnect); + Deliver(x, Data(x, "ping-x")); + Deliver(y, Data(y, "ping-y")); + Send(x, "schema", Encoding.ASCII.GetBytes("pong")); + // Larger than the fragment size: three fragments, one per datagram. + Send(y, "schema", Encoding.ASCII.GetBytes("0123456789abcdefghijklmn")); + Send(x, "latest", Encoding.ASCII.GetBytes("sample")); + // X leaves; Y is undisturbed, and what X sends afterwards belongs to no session. + var late = Data(x, "after-bye"); + Deliver(x, x.Session.CreateDisconnect(Encoding.ASCII.GetBytes("bye"))); + Deliver(x, late); + Send(y, "schema", Encoding.ASCII.GetBytes("still-here")); + // Y reconnects from the same endpoint: a new generation replaces only Y. + y.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 30 }); + Deliver(y, y.Session.CreateConnect(0, Encoding.ASCII.GetBytes("hy2"))); + Send(y, "schema", Encoding.ASCII.GetBytes("new-gen")); + log.Append($"# peers {server.Peers.Count} dropped {server.Stats.PacketsDropped}\n"); + foreach (var client in clients) client.Socket.Dispose(); + return log.ToString(); + } + } +} diff --git a/tests/vectors/rudp-multipeer-server/README.md b/tests/vectors/rudp-multipeer-server/README.md new file mode 100644 index 00000000..21231461 --- /dev/null +++ b/tests/vectors/rudp-multipeer-server/README.md @@ -0,0 +1,13 @@ +# rudp-multipeer-server vectors + +Written only by the C# reference, `tests/GameCult.Networking.Tests/CultNetRudpMultiPeerVectorTests.cs`, with +`CULTNET_WRITE_VECTORS=1`. Every other run replays the scenario and compares it byte for byte; a changed vector is a reviewed +commit. Other runtimes' multi-peer listeners replay the same transcript and are held to the same answers +(`packages/cultnet-py/tests/test_rudp_multipeer_server.py`). + +`transcript.txt` is one listener (`connection_id`, `initial_sequence`, `max_fragment_bytes` and `accept_payload` in its first line) +serving two loopback peers, X and Y. Each line is an action or what it caused: `in ` is a datagram the peer sends the +listener, `send ` is a payload the application sends that peer, and the `out ` lines after it are +every datagram the listener put on the wire in answer, X's before Y's. Peers are named by label because ports are ephemeral. The +scenario covers a Connect, a repeat of an accepted Connect, data and acks, a fragmented send, an unreliable `latest` send, one peer's +Disconnect and the datagram from it that follows, and the other peer's reconnect from the same endpoint. diff --git a/tests/vectors/rudp-multipeer-server/transcript.txt b/tests/vectors/rudp-multipeer-server/transcript.txt new file mode 100644 index 00000000..50f2f64f --- /dev/null +++ b/tests/vectors/rudp-multipeer-server/transcript.txt @@ -0,0 +1,30 @@ +# config connection_id=0x43554c54 initial_sequence=500 max_fragment_bytes=8 accept_payload=616363 +# in | send | out ; peer labels name remote endpoints +in X 434e52300001032b43554c540000000a0000000000000000000000000000000000020700636f6e74726f6c6878 +out X 434e52300002032b43554c54000001f40000000a00000000000000000000000000030700636f6e74726f6c616363 +in Y 434e52300001032b43554c54000000140000000000000000000000000000000000020700636f6e74726f6c6879 +out Y 434e52300002032b43554c54000001f40000001400000000000000000000000000030700636f6e74726f6c616363 +in X 434e52300001032b43554c540000000a0000000000000000000000000000000000020700636f6e74726f6c6878 +out X 434e52300002032b43554c54000001f40000000a00000000000000000000000000030700636f6e74726f6c616363 +in X 434e52300003032a43554c540000000b000001f400000000000000000000000000060600736368656d6170696e672d78 +out X 434e52300004002b43554c54000000000000000b00000001000000000000000000000700636f6e74726f6c +in Y 434e52300003032a43554c5400000015000001f400000000000000000000000000060600736368656d6170696e672d79 +out Y 434e52300004002b43554c54000000000000001500000001000000000000000000000700636f6e74726f6c +send X schema 706f6e67 +out X 434e52300003032a43554c54000001f50000000b00000001000000000000000000040600736368656d61706f6e67 +send Y schema 303132333435363738396162636465666768696a6b6c6d6e +out Y 434e523000030b2a43554c54000001f50000001500000001000100000003000000080600736368656d613031323334353637 +out Y 434e523000030b2a43554c54000001f60000001500000001000100010003000000080600736368656d613839616263646566 +out Y 434e523000030b2a43554c54000001f70000001500000001000100020003000000080600736368656d616768696a6b6c6d6e +send X latest 73616d706c65 +out X 434e52300003042a43554c54000000010000000b000000010000000000000000000606006c617465737473616d706c65 +in X 434e52300007002b43554c5400000000000001f500000001000000000000000000030700636f6e74726f6c627965 +in X 434e52300003032a43554c540000000c000001f500000001000000000000000000090600736368656d6161667465722d627965 +send Y schema 7374696c6c2d68657265 +out Y 434e523000030b2a43554c54000001f80000001500000001000200000002000000080600736368656d617374696c6c2d6865 +out Y 434e523000030b2a43554c54000001f90000001500000001000200010002000000020600736368656d617265 +in Y 434e52300001032b43554c540000001e0000000000000000000000000000000000030700636f6e74726f6c687932 +out Y 434e52300002032b43554c54000001f40000001e00000000000000000000000000030700636f6e74726f6c616363 +send Y schema 6e65772d67656e +out Y 434e52300003032a43554c54000001f50000001e00000000000000000000000000070600736368656d616e65772d67656e +# peers 1 dropped 1