diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index f8a6e5da..19bf1b34 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -9,44 +9,6 @@ on: workflow_dispatch: jobs: - getting-started: - name: ${{ matrix.os }} one-command multiplayer onboarding - runs-on: ${{ matrix.os }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - os: [windows-latest, ubuntu-latest] - - steps: - - name: Check out CultLib - uses: actions/checkout@v4 - with: - path: CultLib - - - name: Check out the matching Eve contract - uses: actions/checkout@v4 - with: - repository: GameCult/Eve - # Eve main is the contract this sample is written against; the codex - # branch this used to pin was never merged and stopped on 2026-08-25. - ref: ${{ vars.EVE_INTEGRATION_REF || 'main' }} - path: Eve - - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 10.0.x - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 24 - - - name: Run the documented one-command checkpoint - shell: pwsh - run: ./CultLib/scripts/verify-eve-getting-started.ps1 -EveRoot ./Eve - reactive-and-resource-lifetime: name: ${{ matrix.os }} reactive semantics and leases runs-on: ${{ matrix.os }} @@ -54,7 +16,8 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest] + # The forge runner is Ubuntu-only: Windows legs run on workflow_dispatch (GitHub or a Windows runner), Ubuntu gates push and PR. + os: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["windows-latest", "ubuntu-latest"]' || '["ubuntu-latest"]') }} steps: - name: Check out CultLib diff --git a/.github/workflows/cultnet-interop.yml b/.github/workflows/cultnet-interop.yml index 9072e0cb..118b496f 100644 --- a/.github/workflows/cultnet-interop.yml +++ b/.github/workflows/cultnet-interop.yml @@ -1,10 +1,9 @@ name: CultNet cross-runtime interop +# workflow_dispatch only: this job is Windows-only (pwsh Kotlin build, setup steps), and the forge +# runner is Ubuntu-only, so a push or pull_request trigger would queue a job that can never run. +# Run it from GitHub or any Windows runner. on: - push: - branches: - - main - pull_request: workflow_dispatch: jobs: diff --git a/CultLib.sln b/CultLib.sln index a8baa383..b72f2d76 100644 --- a/CultLib.sln +++ b/CultLib.sln @@ -35,10 +35,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.WebSock EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GameCult.Networking.WebSockets.Tests", "tests\GameCult.Networking.WebSockets.Tests\GameCult.Networking.WebSockets.Tests.csproj", "{B9A441EC-DADE-4500-B092-C635F3EB296A}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{5D20AA90-6969-D8BD-9DCD-8634F4692FDA}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EveBrowserNetworkSample", "samples\eve-browser-network\EveBrowserNetworkSample.csproj", "{5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "packages", "packages", "{809F86A1-1C4C-B159-0CD4-DF9D33D876CE}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cultmath", "cultmath", "{B48EAE30-152F-01E3-6E31-82B350494D3C}" @@ -231,18 +227,6 @@ Global {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x64.Build.0 = Release|Any CPU {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x86.ActiveCfg = Release|Any CPU {B9A441EC-DADE-4500-B092-C635F3EB296A}.Release|x86.Build.0 = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x64.ActiveCfg = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x64.Build.0 = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x86.ActiveCfg = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Debug|x86.Build.0 = Debug|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|Any CPU.Build.0 = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x64.ActiveCfg = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x64.Build.0 = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x86.ActiveCfg = Release|Any CPU - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB}.Release|x86.Build.0 = Release|Any CPU {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|Any CPU.Build.0 = Debug|Any CPU {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -322,7 +306,6 @@ Global {21F26329-2119-4037-93CC-DD1801A1AB78} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} {7F32D96B-AB5D-4A72-9217-B5DCCFEFC7E5} = {E733CBFF-42AE-4B6A-B5B9-0071ECFB45F1} {B9A441EC-DADE-4500-B092-C635F3EB296A} = {39D316AB-ABB5-49A2-8E83-CAF4C75B48F4} - {5FA81CC7-87B2-4585-9D8E-9BBEC0AD85FB} = {5D20AA90-6969-D8BD-9DCD-8634F4692FDA} {B48EAE30-152F-01E3-6E31-82B350494D3C} = {809F86A1-1C4C-B159-0CD4-DF9D33D876CE} {54D543F7-8EB7-4A11-977E-22D3EE2D8C3F} = {B48EAE30-152F-01E3-6E31-82B350494D3C} {FEF9DA88-2177-4FD0-885D-7096ACFA8A9D} = {B48EAE30-152F-01E3-6E31-82B350494D3C} diff --git a/README.md b/README.md index 6e6de55c..b916cabe 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,8 @@ start with the maintained series. Its first executable package-consumer checkpoint is [`samples/eve-two-runtime`](samples/eve-two-runtime/README.md). The real-process browser continuation is -[`samples/eve-browser-network`](samples/eve-browser-network/README.md). +[`samples/eve-browser-network`](samples/eve-browser-network/README.md); it +needs a sibling Eve checkout (`--eve-root`) and is not in `CultLib.sln`. One documented command runs both layers: ```powershell diff --git a/contracts/cultnet/interop/error-vectors.json b/contracts/cultnet/interop/error-vectors.json new file mode 100644 index 00000000..05422f9e --- /dev/null +++ b/contracts/cultnet/interop/error-vectors.json @@ -0,0 +1,134 @@ +{ + "provenance": "Captured from MessagePack C# 3.1.7 (CultNetErrorMessage/CultNetErrorDetails, Options = Standard.WithSecurity(UntrustedData)); pinned byte-for-byte by packages/cultnet-rs/tests/error_message.rs, whose inline hex equals messagePackHex here (test error_vectors_file_matches_the_inline_csharp_captures). Rust encodes to these exact bytes, so C#-written and Rust-written bytes are one set. Every key is written on the wire, nil included.", + "vectors": [ + { + "name": "selection_invalid_schemas_null", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92473656C656374696F6E5F696E76616C69643A206D757374206E6F7420626520656D707479AB726F7574696E6748696E74C0A4636F6465B173656C656374696F6E5F696E76616C6964A764657461696C7384A56669656C64A7736368656D6173A576616C7565C0A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "selection_invalid: must not be empty", + "routingHint": null, + "code": "selection_invalid", + "details": { + "field": "schemas", + "value": null, + "asOf": null, + "current": null + } + } + }, + { + "name": "selection_invalid_field_value", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92273656C656374696F6E5F696E76616C69643A20626C616E6B206B657920656E747279AB726F7574696E6748696E74C0A4636F6465B173656C656374696F6E5F696E76616C6964A764657461696C7384A56669656C64A46B657973A576616C7565A3202020A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "selection_invalid: blank key entry", + "routingHint": null, + "code": "selection_invalid", + "details": { + "field": "keys", + "value": " ", + "asOf": null, + "current": null + } + } + }, + { + "name": "cursor_stale", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72BD637572736F725F7374616C653A20637572736F72206973207374616C65AB726F7574696E6748696E74C0A4636F6465AC637572736F725F7374616C65A764657461696C7384A56669656C64C0A576616C7565C0A461734F6601A763757272656E7402", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_stale: cursor is stale", + "routingHint": null, + "code": "cursor_stale", + "details": { + "field": null, + "value": null, + "asOf": 1, + "current": 2 + } + } + }, + { + "name": "cursor_invalid", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D923637572736F725F696E76616C69643A2063616E6E6F7420706172736520637572736F72AB726F7574696E6748696E74C0A4636F6465AE637572736F725F696E76616C6964A764657461696C73C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "cursor_invalid: cannot parse cursor", + "routingHint": null, + "code": "cursor_invalid", + "details": null + } + }, + { + "name": "reference_outside_target", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D9317265666572656E63655F6F7574736964655F7461726765743A206F757473696465206465636C6172656420746172676574AB726F7574696E6748696E74C0A4636F6465B87265666572656E63655F6F7574736964655F746172676574A764657461696C73C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "reference_outside_target: outside declared target", + "routingHint": null, + "code": "reference_outside_target", + "details": null + } + }, + { + "name": "variant_unsupported_subscription_ended", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C7384A56669656C64AE737562736372697074696F6E4964A576616C7565A57375622D31A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "variant unsupported here", + "routingHint": null, + "code": "variant_unsupported", + "details": { + "field": "subscriptionId", + "value": "sub-1", + "asOf": null, + "current": null + } + } + }, + { + "name": "variant_unsupported_no_subscription", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72B876617269616E7420756E737570706F727465642068657265AB726F7574696E6748696E74C0A4636F6465B376617269616E745F756E737570706F72746564A764657461696C73C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "variant unsupported here", + "routingHint": null, + "code": "variant_unsupported", + "details": null + } + }, + { + "name": "unowned_schema", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D95C756E6F776E65645F736368656D613A204E6F207368617264206F776E7320736368656D612027736F6D652E736368656D612E763127206B65792027736F6D653A6B6579273B207468652077726974652077617320726566757365642EAB726F7574696E6748696E74C0A4636F6465AE756E6F776E65645F736368656D61A764657461696C7384A56669656C64A8736368656D614964A576616C7565AE736F6D652E736368656D612E7631A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "unowned_schema: No shard owns schema 'some.schema.v1' key 'some:key'; the write was refused.", + "routingHint": null, + "code": "unowned_schema", + "details": { + "field": "schemaId", + "value": "some.schema.v1", + "asOf": null, + "current": null + } + } + }, + { + "name": "write_not_authorized", + "messagePackHex": "85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D92177726974655F6E6F745F617574686F72697A65643A206E6F74206772616E746564AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A8736F6D653A6B6579A461734F66C0A763757272656E74C0", + "message": { + "schemaVersion": "cultnet.error.v0", + "error": "write_not_authorized: not granted", + "routingHint": null, + "code": "write_not_authorized", + "details": { + "field": "recordKey", + "value": "some:key", + "asOf": null, + "current": null + } + } + } + ] +} diff --git a/docs/start/README.md b/docs/start/README.md index 99b66354..42adc5d9 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -16,10 +16,10 @@ this page. | | CultCache | CultMath | Geometry | CultNet | CultMesh | | --- | --- | --- | --- | --- | --- | -| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [package](../../src/GameCult.Networking); snippet not yet | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | -| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [package](../../packages/cultnet-ts/README.md); snippet not yet | [package](../../packages/cultmesh-ts/README.md); snippet not yet | -| **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples); snippet not yet | [package](../../packages/cultmesh-rs); snippet not yet | -| **Python** | [first call, tested](#python-first-call) | not yet | not yet | [package](../../packages/cultnet-py/README.md); snippet not yet | [package](../../packages/cultmesh-py/README.md); snippet not yet | +| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [first call, tested](#c-cultnet-first-call); [package](../../src/GameCult.Networking) | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` | +| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [first call, tested](#typescript-cultnet-first-call); [package](../../packages/cultnet-ts/README.md) | [package](../../packages/cultmesh-ts/README.md); snippet not yet; [recipe: two-client .io game](io-game.md), tested with two live clients, browser page not yet run | +| **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [first call, tested](#rust-cultnet-first-call); [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples) | [package](../../packages/cultmesh-rs); snippet not yet | +| **Python** | [first call, tested](#python-first-call) | not yet | not yet | [first call, tested](#python-cultnet-first-call); [package](../../packages/cultnet-py/README.md) | [package](../../packages/cultmesh-py/README.md); snippet not yet | For the whole picture of what each piece is for, see the @@ -222,6 +222,244 @@ def first_call() -> Item: return reopened.get_required(item_doc, "item:potion") ``` +## CultNet first calls + +CultNet is schema-v0 messages in length-prefixed frames. The first call in each runtime is the same +shape: define a typed document, build a `cultnet.document_put_raw.v0` message carrying its MessagePack +payload, send one frame over a loopback pipe, and apply the received message to a second cache. Each +block is verbatim from the test that runs it; a second test in the same file fails if the block stops +matching. + +### C# CultNet first call + +Source: [`tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs`](../../tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs), +run by `dotnet test tests/GameCult.Networking.Tests --filter CultNetFirstCallTests`. More: `src/GameCult.Networking/README.md`. + +```csharp +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using MessagePack; + +[CultDocument("gamecult.first_call_note", "gamecult.first_call_note.v1")] +[MessagePackObject] +public sealed class Note +{ + [Key(0)] [CultName] public string Title = string.Empty; + [Key(1)] public string Body = string.Empty; +} + +public static class CultNetFirstCall +{ + public static async Task Run() + { + var inbox = new CultCache(); // the receiver's cache + var documents = new CultNetDocumentRegistry(); + var received = new TaskCompletionSource(); + + // Receive: a loopback listener that applies each raw document put to the inbox. + using var server = new TcpFramedCultNetSchemaServer(new TcpListener(IPAddress.Loopback, 0)); + server.OnCultNet(async (put, peer) => + { + await documents.ApplyRawDocumentPutMessageAsync(inbox, put); + received.TrySetResult(true); + }); + + // Send: a typed document as a cultnet.document_put_raw.v0 message. + using var client = new TcpFramedCultNetSchemaClient(); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + var handle = new CultRecordHandle(new CultRecordKey("note:hello")); + client.SendCultNet(documents.CreateRawDocumentPutMessage( + "put-1", handle, new Note { Title = "Hello", Body = "sent over CultNet" })); + + await received.Task.WaitAsync(TimeSpan.FromSeconds(5)); + return inbox.Get(handle.Key); + } +} +``` + +### TypeScript CultNet first call + +Source: [`packages/cultnet-ts/test/first-call.test.ts`](../../packages/cultnet-ts/test/first-call.test.ts), run by +`npm test` in `packages/cultnet-ts` (build `packages/cultcache-ts` first: `npm run build` there). The test imports +from `../src` where the block reads `cultnet-ts`. More: `packages/cultnet-ts/README.md`. + +```ts +import { createServer, connect } from "node:net"; +import { z } from "zod"; +import { CultCache, defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + createTcpFramedCultNetPeer, + defineCultNetDocumentBinding, +} from "cultnet-ts"; + +const noteDocument = defineDocumentType({ + type: "note", + schemaId: "gamecult.first_call_note.v1", + schemaName: "gamecult.first_call_note", + schemaVersion: "gamecult.first_call_note.v1", + schema: z.object({ title: z.string(), body: z.string() }), + name: "title", +}); +const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: noteDocument })]); + +async function firstCall() { + const options = { runtimeId: "first-call", wireContract: "cultnet.schema.v0" } as const; + const inbox = CultCache.builder().withDocumentType(noteDocument).build(); // the receiver's cache + + // Receiver: a loopback listener whose peer applies each document put to the inbox. + const received = new Promise((resolve, reject) => { + const server = createServer((socket) => { + const peer = createTcpFramedCultNetPeer(socket, options); + peer.on("invalidMessage", reject); + peer.on("message", async (message) => { + if (message.schemaVersion !== "cultnet.document_put_raw.v0") return; + await documents.applyRawDocumentPutMessage(inbox, message); + peer.close(); + server.close(); + resolve(); + }); + }).listen(0, "127.0.0.1", () => { + // Sender: connect, build a typed put message, send it. + const { port } = server.address() as { port: number }; + const sender = createTcpFramedCultNetPeer(connect(port, "127.0.0.1"), options); + sender.send( + documents.createRawDocumentPutMessage(documents.get("note")!, "put-1", "note:hello", { + title: "Hello", + body: "sent over CultNet", + }), + ); + received.finally(() => sender.close()); + }); + }); + await received; + return inbox.get(noteDocument, "note:hello"); +} +``` + +### Rust CultNet first call + +Source: [`packages/cultnet-rs/tests/first_call.rs`](../../packages/cultnet-rs/tests/first_call.rs), run by +`cargo test --test first_call` in `packages/cultnet-rs`. More: `packages/cultnet-rs/README.md`. + +```rust +use std::net::{TcpListener, TcpStream}; +use std::thread; + +use cultnet_rs::{ + CultNetDocumentBinding, CultNetDocumentPutOptions, CultNetDocumentRegistry, CultNetWireContract, + TcpFramedTransportConnection, create_tcp_framed_transport_profile, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, +}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "note")] +struct Note { + #[cultcache(key = 0)] + title: String, + #[cultcache(key = 1)] + body: String, +} + +fn first_call() -> anyhow::Result { + let mut documents = CultNetDocumentRegistry::new(); + documents.register(CultNetDocumentBinding::for_entry_with_schema_id::( + "gamecult.first_call_note.v1".to_string(), + "gamecult.first_call_note.v1".to_string(), + ))?; + let profile = |runtime_id| create_tcp_framed_transport_profile(runtime_id, Default::default()); + + // Receiver: a loopback listener that applies the document put it receives to its own cache. + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let receiver = thread::spawn({ + let (documents, profile) = (documents.clone(), profile("receiver")); + move || -> anyhow::Result { + let (stream, _) = listener.accept()?; + let frame = TcpFramedTransportConnection::new(stream, profile).receive()?; + let message = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + let mut inbox = CultCache::new(); + inbox.register_entry_type::()?; + documents.apply_raw_document_put_message::(&mut inbox, &message) + } + }); + + // Sender: build a typed put message, encode it for the wire, send one frame. + let put = documents.create_raw_document_put_message( + "put-1", + "note:hello", + &Note { title: "Hello".to_string(), body: "sent over CultNet".to_string() }, + CultNetDocumentPutOptions::default(), + )?; + let bytes = encode_cultnet_message_to_vec(&put, CultNetWireContract::CultNetSchemaV0)?; + TcpFramedTransportConnection::new(TcpStream::connect(("127.0.0.1", port))?, profile("sender")) + .send("schema", &bytes)?; + + receiver.join().expect("receiver thread") +} +``` + +### Python CultNet first call + +Source: [`packages/cultnet-py/tests/test_first_call.py`](../../packages/cultnet-py/tests/test_first_call.py), run by +`python -m unittest tests.test_first_call` in `packages/cultnet-py` with both `src` and `../cultcache-py/src` on +`PYTHONPATH` and `msgpack` installed. The socket pair stands in for a TCP connection. More: `packages/cultnet-py/README.md`. + +```python +import socket +from dataclasses import asdict, dataclass + +from cultcache_py import CultCache, define_document_type +from cultnet_py import ( + apply_raw_document_record, + document_put_raw, + encode_frame, + parse_message, + read_frame, + schema_document_map, +) + + +@dataclass +class Note: + title: str + body: str + + +note_doc = define_document_type( + "note", + encode=lambda note: asdict(note), + decode=lambda raw: Note(**raw), + name="title", +) + + +def first_call() -> Note: + sender, receiver = socket.socketpair() # an in-process pipe; use socket.create_connection for TCP + + # Send: a typed document as a cultnet.document_put_raw.v0 message in one length-prefixed frame. + put = document_put_raw( + message_id="put-1", + key="note:hello", + schema_id=note_doc.catalog_entry().schema_id, + stored_at="2026-01-01T00:00:00Z", + payload=note_doc.encode_payload(Note("Hello", "sent over CultNet")), + ) + sender.sendall(encode_frame(put.to_bytes())) + + # Receive: read the frame, parse the message, apply its record to a second cache. + message = parse_message(read_frame(receiver.makefile("rb"))) + assert message.schema_version == "cultnet.document_put_raw.v0" + inbox = CultCache.builder().register_document_type(note_doc).build() + apply_raw_document_record(inbox, schema_document_map([note_doc]), message.body["document"]) + sender.close() + receiver.close() + return inbox.get_required(note_doc, "note:hello") +``` + ## Where next - Smallest durable mesh path: `src/GameCult.Mesh/docs/durable-node-quickstart.md`. diff --git a/docs/start/io-game.md b/docs/start/io-game.md new file mode 100644 index 00000000..4781d9bf --- /dev/null +++ b/docs/start/io-game.md @@ -0,0 +1,196 @@ +# Recipe: a two-client .io game over CultCache + CultMesh (TypeScript) + +A tiny arena. One process is the authority: it holds a single `io.arena_state` document in a CultCache +and serves it over CultMesh. Each client discovers the authority through the Odin catalog, leases the +document, and invokes `join`, `move` and `leave` operations. Two clients see each other join, move and +leave because both lease the same document. + +## What ran, what did not + +| Piece | Status | +| --- | --- | +| Two clients, real WebSockets, real `cultmesh-browser` client, one arena: join, see each other, move, leave | **tested** (`arena.test.ts`) | +| Edge clamp, and the source below matching the files that ran | **tested** | +| Browser bundle builds and the page is served | **tested** (HTTP fetch only) | +| The page running in a real browser, two windows | **not yet**: no browser on the host that built this recipe | +| Run by CI / `npm run test:ts` | **not yet**: the recipe has its own `package.json` (for `ws`) outside the workspaces; run it by hand as below | +| Authority state on disk (`.cc` file) | **not yet**: the arena cache is in memory | +| Remote play: Odin roots and a signed route | **not yet**: the client uses `local-development` trust, loopback only | +| Dropped players: a closed tab that never sent `leave` stays in the arena | **not yet**: operations do not carry a connection identity to the authority | +| `CultCache` in the browser | **not yet**: the `@gamecult/cultcache-ts` entry imports Node built-ins, so the client keeps the decoded document in a variable and only `./document` is imported | + +No `cultmesh-ts` or `cultnet-ts` TypeScript authority server exists. The server below is the CultNet +message handling a provider owes, written in this recipe with the `ws` package. + +## Run it + +``` +npm ci --ignore-scripts # repo root +(cd packages/cultnet-ts && npm run build) # then cultcache-ts, cultmesh-browser, same command +cd docs/start/io-game && npm install +npm test # node --test arena.test.ts, Node 24 +node serve.ts # http://127.0.0.1:8080, open it twice +``` + +Tested on Node v24.14, which runs `.ts` files directly (no type check; `tsc` is not part of this recipe). + +## schema.ts + +```ts +import { defineDocumentType } from "@gamecult/cultcache-ts/document"; + +export interface Player { x: number; y: number } +export interface ArenaState { players: Record } + +// The one shared document: every player's position. The server owns it; clients lease it. +export const arenaDocument = defineDocumentType({ + type: "io.arena_state", + schemaId: "io.arena_state.v1", + schema: { parse: (value: unknown) => value as ArenaState }, +}); + +export const arenaKey = "arena:main"; +export const verseId = "io.arena"; +export const authorityRuntimeId = "io.arena-daemon"; +export const arenaSize = 100; +``` + +## server.ts + +```ts +import { createServer, type RequestListener, type Server } from "node:http"; +import { CultCache } from "@gamecult/cultcache-ts"; +import { defineDocumentRegistry } from "@gamecult/cultcache-ts/document"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "cultnet-ts/contracts"; +import { WebSocketServer, type WebSocket } from "ws"; +import { arenaDocument, arenaKey, arenaSize, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +const bytesToBase64 = (b: Uint8Array) => Buffer.from(b).toString("base64"); +const base64ToBytes = (s: string) => new Uint8Array(Buffer.from(s, "base64")); + +// One process plays both CultMesh roles: Odin (/odin tells clients where the authority is) and the authority (/mesh). +export async function startArena(port = 0, host = "127.0.0.1", onRequest?: RequestListener) { + const cache = CultCache.builder().withRegistry(defineDocumentRegistry(arenaDocument)).build(); + await cache.put(arenaDocument, arenaKey, { players: {} }); + const http: Server = createServer(onRequest ?? ((_, res) => { res.writeHead(404).end(); })); + await new Promise(resolve => http.listen(port, host, resolve)); + const endpoint = `ws://${host}:${(http.address() as { port: number }).port}`; + + const subscribers = new Map(); + const send = (ws: WebSocket, m: CultNetMessage) => ws.send(encodeWireValue(encodeCultNetMessageForWire(m))); + const record = () => ({ + schemaId: arenaDocument.schemaId, schemaName: arenaDocument.type, schemaVersion: arenaDocument.schemaId, + schemaContentHash: arenaDocument.schemaId, recordKey: arenaKey, storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encodeWireValue(cache.getRequired(arenaDocument, arenaKey)) as never, + sourceRuntimeId: authorityRuntimeId, + }); + const publish = async (next: ArenaState) => { + await cache.put(arenaDocument, arenaKey, next); + for (const [ws, ids] of subscribers) for (const subscriptionId of ids) { + send(ws, { schemaVersion: "cultnet.database_change_raw.v0", messageId: crypto.randomUUID(), subscriptionId, changeKind: "updated", document: record() }); + } + }; + + let nextId = 0; + const operations: Record Promise> = { + join: async () => { + const id = `p${++nextId}`; + const state = cache.getRequired(arenaDocument, arenaKey); + await publish({ players: { ...state.players, [id]: { x: Math.floor(Math.random() * arenaSize), y: Math.floor(Math.random() * arenaSize) } } }); + return { id }; + }, + move: async ({ id, dx, dy }: { id: string; dx: number; dy: number }) => { + const state = cache.getRequired(arenaDocument, arenaKey); + const me = state.players[id]; + if (!me) return { ok: false }; + const clamp = (v: number) => Math.max(0, Math.min(arenaSize, v)); + await publish({ players: { ...state.players, [id]: { x: clamp(me.x + Math.sign(dx)), y: clamp(me.y + Math.sign(dy)) } } }); + return { ok: true }; + }, + leave: async ({ id }: { id: string }) => { + const { [id]: _gone, ...rest } = cache.getRequired(arenaDocument, arenaKey).players; + await publish({ players: rest }); + return { ok: true }; + }, + }; + + const wss = new WebSocketServer({ server: http }); + wss.on("connection", (ws, req) => { + const odin = req.url === "/odin"; + ws.on("close", () => subscribers.delete(ws)); + ws.on("message", async raw => { + const m = parseCultNetMessage(decodeWireValue(new Uint8Array(raw as Buffer))); + if (odin && m.schemaVersion === "cultmesh.verse_catalog_request.v0") { + send(ws, { + schemaVersion: "cultmesh.verse_catalog_response.v0", messageId: m.messageId, + verses: [{ + verseId, displayName: "Arena", authorityModel: "ServerAuthoritative", + compatibility: { transportVersion: "cultmesh.v1", rulesHash: "arena-v1", compatibleVerseIds: [], requiredPluginIds: [], optionalPluginIds: [] }, + discoveryEndpoints: [`${endpoint}/mesh`], authorityRuntimeIds: [authorityRuntimeId], + authorityRoutes: [{ authorityRuntimeId, endpoint: `${endpoint}/mesh`, protocolIds: ["cultmesh.documents.v1"], priority: 0, generation: "arena-1" }], + }], + }); + } else if (m.schemaVersion === "cultmesh.session_open.v2") { + send(ws, { schemaVersion: "cultmesh.session_accepted.v2", messageId: m.messageId, accepted: true, verseId: m.verseId, authorityRuntimeId, + protocolId: m.protocolId, routeGeneration: m.routeGeneration, clientNonce: m.clientNonce, providerKeyId: "", providerSignature: "" }); + } else if (m.schemaVersion === "cultnet.database_subscribe.v0") { + subscribers.set(ws, [...(subscribers.get(ws) ?? []), m.subscriptionId]); + send(ws, { schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: m.messageId, documents: [record()] }); + } else if (m.schemaVersion === "cultnet.database_unsubscribe.v0") { + subscribers.set(ws, (subscribers.get(ws) ?? []).filter(id => id !== m.subscriptionId)); + } else if (m.schemaVersion === "cultnet.operation_request.v0") { + const handler = operations[m.operation]; + const receipt = handler ? await handler(decodeWireValue(base64ToBytes(m.payload))) : undefined; + send(ws, { + schemaVersion: "cultnet.operation_response.v0", messageId: m.messageId, serviceId: m.serviceId, operation: m.operation, + status: handler ? "accepted" : "invalid", payloadSchema: "io.receipt.v1", payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encodeWireValue(receipt ?? { ok: false })), sourceRuntimeId: authorityRuntimeId, + }); + } + }); + }); + + return { + odinUrl: `${endpoint}/odin`, + state: () => cache.getRequired(arenaDocument, arenaKey), + close: async () => { for (const c of wss.clients) c.terminate(); wss.close(); await new Promise(r => http.close(r)); }, + }; +} +``` + +## client.ts + +```ts +import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "cultmesh-browser"; +import { arenaDocument, arenaKey, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +// Runs unchanged in a browser or in Node 22+: both have a global WebSocket. +export async function joinArena(odinUrl: string, runtimeId: string, onState: (state: ArenaState, myId: string) => void) { + const rendezvous = new CultMeshBrowserOdinRendezvous({ endpoints: [odinUrl], runtimeId: `${runtimeId}.discovery` }); + const mesh = await CultMeshBrowserClient.connect({ + verseId, authorityRuntimeId, runtimeId, rendezvous, + trust: { mode: "local-development" }, // loopback only; remote Verses need Odin roots + }); + let latest: ArenaState | undefined; // CultCache itself pulls in Node built-ins, so the browser client keeps the decoded document + const call = async (operation: string, payload: object) => + decodeCultNetOperationPayload<{ id?: string }>(await mesh.invoke({ serviceId: "io.arena", operation, payloadSchema: "io.request.v1", payload })); + + const lease = await mesh.leaseRawDocument({ schemaId: arenaDocument.schemaId, recordKey: arenaKey }); + const { id } = await call("join", {}); + lease.watch(record => { + if (!record) return; + latest = decodeCultNetPayload(record, arenaDocument); + onState(latest, id!); + }); + return { + id: id!, + move: (dx: number, dy: number) => call("move", { id, dx, dy }), + state: () => latest, + leave: async () => { await call("leave", { id }); lease.dispose(); await mesh.dispose(); }, + }; +} +``` + +`browser.ts`, `index.html` and `serve.ts` in [`io-game/`](io-game/) draw the canvas and bundle the client with esbuild. diff --git a/docs/start/io-game/arena.test.ts b/docs/start/io-game/arena.test.ts new file mode 100644 index 00000000..67049907 --- /dev/null +++ b/docs/start/io-game/arena.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { joinArena } from "./client.ts"; +import { startArena } from "./server.ts"; +import { arenaSize, type ArenaState } from "./schema.ts"; + +const until = async (cond: () => boolean) => { + for (let i = 0; i < 200 && !cond(); i++) await new Promise(r => setTimeout(r, 10)); + assert.ok(cond(), "condition not reached in 2s"); +}; + +test("two clients join one arena, see each other, and see each other move", async () => { + const arena = await startArena(); + const seenByA: ArenaState[] = []; + const seenByB: ArenaState[] = []; + const a = await joinArena(arena.odinUrl, "client-a", s => seenByA.push(structuredClone(s))); + const b = await joinArena(arena.odinUrl, "client-b", s => seenByB.push(structuredClone(s))); + try { + assert.notEqual(a.id, b.id); + await until(() => Object.keys(a.state()?.players ?? {}).length === 2); + await until(() => Object.keys(b.state()?.players ?? {}).length === 2); + + const before = structuredClone(b.state()!.players[a.id]!); + // Moves clamp at the arena edge, so pick a direction that always has room to move. + const dx = before.x > 0 ? -1 : 1; + await a.move(dx, 0); + await until(() => b.state()!.players[a.id]!.x !== before.x); + assert.equal(b.state()!.players[a.id]!.x, before.x + dx); + assert.deepEqual(a.state(), arena.state()); + assert.deepEqual(b.state(), arena.state()); + + await b.leave(); + await until(() => !(a.id !== b.id && b.id in a.state()!.players)); + assert.deepEqual(Object.keys(a.state()!.players), [a.id]); + } finally { + await a.leave().catch(() => undefined); + await arena.close(); + } +}); + +test("a player cannot walk out of the arena", async () => { + const arena = await startArena(); + const a = await joinArena(arena.odinUrl, "client-a", () => undefined); + try { + for (let i = 0; i <= arenaSize; i++) await a.move(-1, 0); + assert.equal(arena.state().players[a.id]!.x, 0); + } finally { + await a.leave(); + await arena.close(); + } +}); + +test("serve.ts bundles the browser client and serves the page", async () => { + const { serveArena } = await import("./serve.ts"); + const site = await serveArena(0); + try { + const html = await (await fetch(site.url)).text(); + assert.match(html, / { + const page = readFileSync(new URL("../io-game.md", import.meta.url), "utf8").replace(/\r\n/g, "\n"); + for (const file of ["schema.ts", "server.ts", "client.ts"]) { + const text = readFileSync(new URL(`./${file}`, import.meta.url), "utf8").replace(/\r\n/g, "\n"); + assert.ok(page.includes(text), `docs/start/io-game.md no longer shows ${file} verbatim`); + } +}); diff --git a/docs/start/io-game/browser.ts b/docs/start/io-game/browser.ts new file mode 100644 index 00000000..4120285f --- /dev/null +++ b/docs/start/io-game/browser.ts @@ -0,0 +1,17 @@ +import { joinArena } from "./client.ts"; +import { arenaSize, type ArenaState } from "./schema.ts"; + +const canvas = document.querySelector("canvas")!; +const ctx = canvas.getContext("2d")!; +const scale = canvas.width / arenaSize; + +const game = await joinArena(`ws://${location.host}/odin`, `browser-${crypto.randomUUID()}`, (state: ArenaState, me: string) => { + ctx.clearRect(0, 0, canvas.width, canvas.height); + for (const [id, p] of Object.entries(state.players)) { + ctx.fillStyle = id === me ? "#f5c542" : "#6aa9ff"; + ctx.fillRect(p.x * scale, p.y * scale, scale * 2, scale * 2); + } +}); +const keys: Record = { ArrowLeft: [-1, 0], ArrowRight: [1, 0], ArrowUp: [0, -1], ArrowDown: [0, 1] }; +addEventListener("keydown", e => { const d = keys[e.key]; if (d) void game.move(...d); }); +addEventListener("pagehide", () => void game.leave()); diff --git a/docs/start/io-game/client.ts b/docs/start/io-game/client.ts new file mode 100644 index 00000000..8493e23c --- /dev/null +++ b/docs/start/io-game/client.ts @@ -0,0 +1,28 @@ +import { CultMeshBrowserClient, CultMeshBrowserOdinRendezvous, decodeCultNetOperationPayload, decodeCultNetPayload } from "cultmesh-browser"; +import { arenaDocument, arenaKey, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +// Runs unchanged in a browser or in Node 22+: both have a global WebSocket. +export async function joinArena(odinUrl: string, runtimeId: string, onState: (state: ArenaState, myId: string) => void) { + const rendezvous = new CultMeshBrowserOdinRendezvous({ endpoints: [odinUrl], runtimeId: `${runtimeId}.discovery` }); + const mesh = await CultMeshBrowserClient.connect({ + verseId, authorityRuntimeId, runtimeId, rendezvous, + trust: { mode: "local-development" }, // loopback only; remote Verses need Odin roots + }); + let latest: ArenaState | undefined; // CultCache itself pulls in Node built-ins, so the browser client keeps the decoded document + const call = async (operation: string, payload: object) => + decodeCultNetOperationPayload<{ id?: string }>(await mesh.invoke({ serviceId: "io.arena", operation, payloadSchema: "io.request.v1", payload })); + + const lease = await mesh.leaseRawDocument({ schemaId: arenaDocument.schemaId, recordKey: arenaKey }); + const { id } = await call("join", {}); + lease.watch(record => { + if (!record) return; + latest = decodeCultNetPayload(record, arenaDocument); + onState(latest, id!); + }); + return { + id: id!, + move: (dx: number, dy: number) => call("move", { id, dx, dy }), + state: () => latest, + leave: async () => { await call("leave", { id }); lease.dispose(); await mesh.dispose(); }, + }; +} diff --git a/docs/start/io-game/index.html b/docs/start/io-game/index.html new file mode 100644 index 00000000..bcc77648 --- /dev/null +++ b/docs/start/io-game/index.html @@ -0,0 +1,6 @@ + + +arena + +

Arrow keys. Open this page in a second window.

+ diff --git a/docs/start/io-game/package-lock.json b/docs/start/io-game/package-lock.json new file mode 100644 index 00000000..48afe0a7 --- /dev/null +++ b/docs/start/io-game/package-lock.json @@ -0,0 +1,64 @@ +{ + "name": "cultlib-start-io-game", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cultlib-start-io-game", + "dependencies": { + "ws": "^8.18.0" + }, + "devDependencies": { + "@types/ws": "^8.5.0" + } + }, + "node_modules/@types/node": { + "version": "26.6.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz", + "integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/ws": { + "version": "8.18.2", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.2.tgz", + "integrity": "sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ws": { + "version": "8.22.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.1.tgz", + "integrity": "sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/docs/start/io-game/package.json b/docs/start/io-game/package.json new file mode 100644 index 00000000..d8f1281f --- /dev/null +++ b/docs/start/io-game/package.json @@ -0,0 +1,9 @@ +{ + "name": "cultlib-start-io-game", + "private": true, + "type": "module", + "description": "docs/start recipe: a tiny two-client browser game over CultCache + CultMesh (TypeScript).", + "scripts": { "test": "node --test arena.test.ts" }, + "dependencies": { "ws": "^8.18.0" }, + "devDependencies": { "@types/ws": "^8.5.0" } +} diff --git a/docs/start/io-game/schema.ts b/docs/start/io-game/schema.ts new file mode 100644 index 00000000..d0c1a59b --- /dev/null +++ b/docs/start/io-game/schema.ts @@ -0,0 +1,16 @@ +import { defineDocumentType } from "@gamecult/cultcache-ts/document"; + +export interface Player { x: number; y: number } +export interface ArenaState { players: Record } + +// The one shared document: every player's position. The server owns it; clients lease it. +export const arenaDocument = defineDocumentType({ + type: "io.arena_state", + schemaId: "io.arena_state.v1", + schema: { parse: (value: unknown) => value as ArenaState }, +}); + +export const arenaKey = "arena:main"; +export const verseId = "io.arena"; +export const authorityRuntimeId = "io.arena-daemon"; +export const arenaSize = 100; diff --git a/docs/start/io-game/serve.ts b/docs/start/io-game/serve.ts new file mode 100644 index 00000000..82bf20be --- /dev/null +++ b/docs/start/io-game/serve.ts @@ -0,0 +1,17 @@ +import { readFileSync } from "node:fs"; +import { build } from "esbuild"; +import { startArena } from "./server.ts"; + +// Serves the page, the bundled browser client, and the arena's WebSocket on one port. +export async function serveArena(port = 8080, host = "127.0.0.1") { + const bundle = await build({ entryPoints: [new URL("./browser.ts", import.meta.url).pathname], bundle: true, format: "esm", target: "es2022", write: false }); + const html = readFileSync(new URL("./index.html", import.meta.url)); + const arena = await startArena(port, host, (req, res) => { + if (req.url === "/") res.writeHead(200, { "content-type": "text/html" }).end(html); + else if (req.url === "/browser.js") res.writeHead(200, { "content-type": "text/javascript" }).end(bundle.outputFiles[0]!.contents); + else res.writeHead(404).end(); + }); + return { url: arena.odinUrl.replace("ws:", "http:").replace("/odin", ""), close: arena.close }; +} + +if (import.meta.main) console.log(`arena at ${(await serveArena()).url}`); diff --git a/docs/start/io-game/server.ts b/docs/start/io-game/server.ts new file mode 100644 index 00000000..39a1a618 --- /dev/null +++ b/docs/start/io-game/server.ts @@ -0,0 +1,100 @@ +import { createServer, type RequestListener, type Server } from "node:http"; +import { CultCache } from "@gamecult/cultcache-ts"; +import { defineDocumentRegistry } from "@gamecult/cultcache-ts/document"; +import { decodeWireValue, encodeWireValue } from "cultnet-ts/wire"; +import { encodeCultNetMessageForWire, parseCultNetMessage, type CultNetMessage } from "cultnet-ts/contracts"; +import { WebSocketServer, type WebSocket } from "ws"; +import { arenaDocument, arenaKey, arenaSize, authorityRuntimeId, verseId, type ArenaState } from "./schema.ts"; + +const bytesToBase64 = (b: Uint8Array) => Buffer.from(b).toString("base64"); +const base64ToBytes = (s: string) => new Uint8Array(Buffer.from(s, "base64")); + +// One process plays both CultMesh roles: Odin (/odin tells clients where the authority is) and the authority (/mesh). +export async function startArena(port = 0, host = "127.0.0.1", onRequest?: RequestListener) { + const cache = CultCache.builder().withRegistry(defineDocumentRegistry(arenaDocument)).build(); + await cache.put(arenaDocument, arenaKey, { players: {} }); + const http: Server = createServer(onRequest ?? ((_, res) => { res.writeHead(404).end(); })); + await new Promise(resolve => http.listen(port, host, resolve)); + const endpoint = `ws://${host}:${(http.address() as { port: number }).port}`; + + const subscribers = new Map(); + const send = (ws: WebSocket, m: CultNetMessage) => ws.send(encodeWireValue(encodeCultNetMessageForWire(m))); + const record = () => ({ + schemaId: arenaDocument.schemaId, schemaName: arenaDocument.type, schemaVersion: arenaDocument.schemaId, + schemaContentHash: arenaDocument.schemaId, recordKey: arenaKey, storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encodeWireValue(cache.getRequired(arenaDocument, arenaKey)) as never, + sourceRuntimeId: authorityRuntimeId, + }); + const publish = async (next: ArenaState) => { + await cache.put(arenaDocument, arenaKey, next); + for (const [ws, ids] of subscribers) for (const subscriptionId of ids) { + send(ws, { schemaVersion: "cultnet.database_change_raw.v0", messageId: crypto.randomUUID(), subscriptionId, changeKind: "updated", document: record() }); + } + }; + + let nextId = 0; + const operations: Record Promise> = { + join: async () => { + const id = `p${++nextId}`; + const state = cache.getRequired(arenaDocument, arenaKey); + await publish({ players: { ...state.players, [id]: { x: Math.floor(Math.random() * arenaSize), y: Math.floor(Math.random() * arenaSize) } } }); + return { id }; + }, + move: async ({ id, dx, dy }: { id: string; dx: number; dy: number }) => { + const state = cache.getRequired(arenaDocument, arenaKey); + const me = state.players[id]; + if (!me) return { ok: false }; + const clamp = (v: number) => Math.max(0, Math.min(arenaSize, v)); + await publish({ players: { ...state.players, [id]: { x: clamp(me.x + Math.sign(dx)), y: clamp(me.y + Math.sign(dy)) } } }); + return { ok: true }; + }, + leave: async ({ id }: { id: string }) => { + const { [id]: _gone, ...rest } = cache.getRequired(arenaDocument, arenaKey).players; + await publish({ players: rest }); + return { ok: true }; + }, + }; + + const wss = new WebSocketServer({ server: http }); + wss.on("connection", (ws, req) => { + const odin = req.url === "/odin"; + ws.on("close", () => subscribers.delete(ws)); + ws.on("message", async raw => { + const m = parseCultNetMessage(decodeWireValue(new Uint8Array(raw as Buffer))); + if (odin && m.schemaVersion === "cultmesh.verse_catalog_request.v0") { + send(ws, { + schemaVersion: "cultmesh.verse_catalog_response.v0", messageId: m.messageId, + verses: [{ + verseId, displayName: "Arena", authorityModel: "ServerAuthoritative", + compatibility: { transportVersion: "cultmesh.v1", rulesHash: "arena-v1", compatibleVerseIds: [], requiredPluginIds: [], optionalPluginIds: [] }, + discoveryEndpoints: [`${endpoint}/mesh`], authorityRuntimeIds: [authorityRuntimeId], + authorityRoutes: [{ authorityRuntimeId, endpoint: `${endpoint}/mesh`, protocolIds: ["cultmesh.documents.v1"], priority: 0, generation: "arena-1" }], + }], + }); + } else if (m.schemaVersion === "cultmesh.session_open.v2") { + send(ws, { schemaVersion: "cultmesh.session_accepted.v2", messageId: m.messageId, accepted: true, verseId: m.verseId, authorityRuntimeId, + protocolId: m.protocolId, routeGeneration: m.routeGeneration, clientNonce: m.clientNonce, providerKeyId: "", providerSignature: "" }); + } else if (m.schemaVersion === "cultnet.database_subscribe.v0") { + subscribers.set(ws, [...(subscribers.get(ws) ?? []), m.subscriptionId]); + send(ws, { schemaVersion: "cultnet.snapshot_response_raw.v0", messageId: m.messageId, documents: [record()] }); + } else if (m.schemaVersion === "cultnet.database_unsubscribe.v0") { + subscribers.set(ws, (subscribers.get(ws) ?? []).filter(id => id !== m.subscriptionId)); + } else if (m.schemaVersion === "cultnet.operation_request.v0") { + const handler = operations[m.operation]; + const receipt = handler ? await handler(decodeWireValue(base64ToBytes(m.payload))) : undefined; + send(ws, { + schemaVersion: "cultnet.operation_response.v0", messageId: m.messageId, serviceId: m.serviceId, operation: m.operation, + status: handler ? "accepted" : "invalid", payloadSchema: "io.receipt.v1", payloadEncoding: "messagepack-base64", + payload: bytesToBase64(encodeWireValue(receipt ?? { ok: false })), sourceRuntimeId: authorityRuntimeId, + }); + } + }); + }); + + return { + odinUrl: `${endpoint}/odin`, + state: () => cache.getRequired(arenaDocument, arenaKey), + close: async () => { for (const c of wss.clients) c.terminate(); wss.close(); await new Promise(r => http.close(r)); }, + }; +} diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index d2f76628..b460ab05 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +### Changed + +- `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import. +- The single-file store's `push`, `push_all` and `delete` take the cross-process lock on `.lock` (flock; LockFileEx over the whole file on Windows) across the read and the replace, waiting up to `lock_timeout` seconds (30 by default, `TimeoutError` after), and exclude the C#, Rust and TypeScript runtimes' writers. The staged file is `.tmp--`, fsynced before the replace, with a directory fsync after it off Windows. A fixed `.tmp` is no longer used. + ### Breaking - A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 91d09ab8..23849713 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -6,6 +6,9 @@ from collections.abc import Mapping, Sequence, Set from itertools import chain from typing import Any, Callable, Generic, Iterable, TypeVar +import msgpack +from msgpack.exceptions import StackError as _MsgpackStackError + from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember from .leaves import Float32, SlotCodec from .leaves import any as _any_leaf @@ -24,12 +27,7 @@ or skipped, the outermost one is depth 1; depth 256 is accepted and 257 refused, MAX_SLOT_KEY = 65535 """The largest slot key a record may define: the largest MessagePack array16 length.""" -try: - from msgpack.exceptions import StackError as _MsgpackStackError - - _STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,) -except ModuleNotFoundError: - _STACK_FAULTS = () +_STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,) _CONTAINERS = (Mapping, Sequence, Set) @@ -85,22 +83,10 @@ def _json_payload_decoder(payload: bytes) -> Any: return json.loads(payload.decode("utf-8")) -def _msgpack() -> Any: - try: - import msgpack # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "MessagePack payload encoding requires the optional 'msgpack' dependency. " - "Install with: python -m pip install cultcache-py[msgpack]" - ) from exc - return msgpack - - def _pack_tree(tree: Any) -> bytes: """The bytes of the tree the leaves produced. `packb` writes everything but a `Float32`, which it refuses with a TypeError; only then does an iterative walk pack each array and map header and each value, a `Float32` through a single-float packer.""" - msgpack = _msgpack() try: return msgpack.packb(tree, use_bin_type=True) # type: ignore[no-any-return] except TypeError: @@ -125,13 +111,6 @@ def _pack_tree(tree: Any) -> bytes: def _messagepack_payload_decoder(payload: bytes) -> Any: - try: - import msgpack # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "MessagePack payload decoding requires the optional 'msgpack' dependency. " - "Install with: python -m pip install cultcache-py[msgpack]" - ) from exc return msgpack.unpackb(payload, raw=False) diff --git a/packages/cultcache-py/src/cultcache_py/leaves.py b/packages/cultcache-py/src/cultcache_py/leaves.py index 2d76078e..71bc6204 100644 --- a/packages/cultcache-py/src/cultcache_py/leaves.py +++ b/packages/cultcache-py/src/cultcache_py/leaves.py @@ -16,10 +16,7 @@ from collections.abc import Mapping from dataclasses import dataclass from typing import Any, Callable, NoReturn -try: - from msgpack import ExtType as _ExtType -except ModuleNotFoundError: - _ExtType = () # type: ignore[assignment,misc] +from msgpack import ExtType as _ExtType _bool, _bytes, _list, _int, _float, _str = bool, bytes, list, int, float, str _BYTE_LIKE = (_bytes, bytearray, memoryview) diff --git a/packages/cultcache-py/src/cultcache_py/stores.py b/packages/cultcache-py/src/cultcache_py/stores.py index c9c76b06..77d671ad 100644 --- a/packages/cultcache-py/src/cultcache_py/stores.py +++ b/packages/cultcache-py/src/cultcache_py/stores.py @@ -3,9 +3,15 @@ import base64 import json import os +import sys import threading +import time +import uuid +from contextlib import contextmanager from pathlib import Path -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Iterator, TypeVar + +import msgpack from .backing_store import ( CultCacheEnvelope, @@ -21,6 +27,8 @@ STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3" _READABLE_STORE_FORMATS = (STORE_FORMAT_VERSION, STORE_FORMAT_ELEMENT_IDS) _STORE_FORMAT_PREFIX = "cultcache.store." _PERSISTED_RECORD_SLOTS = 4 +_LOCK_POLL_SECONDS = 0.01 +LOCK_TIMEOUT_SECONDS = 30.0 _Contents = TypeVar("_Contents") @@ -114,14 +122,14 @@ class JsonLinesBackingStore: class SingleFileMessagePackBackingStore: - def __init__(self, path: str | os.PathLike[str]) -> None: + def __init__(self, path: str | os.PathLike[str], lock_timeout: float = LOCK_TIMEOUT_SECONDS) -> None: self.path = Path(path) + self.lock_timeout = lock_timeout self._lock = threading.RLock() # The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker. self._format = STORE_FORMAT_VERSION def pull_all(self) -> list[CultCacheEnvelope]: - msgpack = self._msgpack() with self._lock: # An empty file is not a store (no writer leaves one), so it is decoded and refused. # The disk decides the header: a file that is gone is not marked. @@ -135,39 +143,34 @@ class SingleFileMessagePackBackingStore: if snapshot is not None: self._format, envelopes = snapshot return envelopes - return _decode_legacy_envelopes(decoded, msgpack) + return _decode_legacy_envelopes(decoded) except Exception as exc: raise StoreUnreadableError(self.path, exc) from exc def push(self, envelope: CultCacheEnvelope) -> None: - with self._lock: + with self._write_transaction(): self._replace_all([envelope], self.pull_all()) def push_all(self, envelopes: list[CultCacheEnvelope]) -> None: - with self._lock: + with self._write_transaction(): self._replace_all(list(envelopes), self.pull_all()) def delete(self, type: str, key: str) -> None: - with self._lock: + with self._write_transaction(): self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)]) - def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: - msgpack = self._msgpack() - self.path.parent.mkdir(parents=True, exist_ok=True) - temp = self.path.with_suffix(self.path.suffix + ".tmp") - temp.write_bytes(msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) - temp.replace(self.path) + @contextmanager + def _write_transaction(self) -> Iterator[None]: + """The thread lock, then the cross-process lock on `.lock`, held across the read and the replace so two + processes never rewrite from the same stale read.""" + with self._lock: + self.path.parent.mkdir(parents=True, exist_ok=True) + with store_write_lock(self.path, self.lock_timeout): + yield + + def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None: + commit_store_file(self.path, msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True)) - @staticmethod - def _msgpack() -> Any: - try: - import msgpack # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "SingleFileMessagePackBackingStore requires the optional 'msgpack' dependency. " - "Install with: python -m pip install cultcache-py[msgpack]" - ) from exc - return msgpack def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]: @@ -261,6 +264,85 @@ def _encode_snapshot( ] return [format_version, catalog, records] +def commit_store_file(path: Path, data: bytes) -> None: + """Writes `data` to a unique staged file beside the store, fsyncs it, replaces the store with it and, off Windows, + fsyncs the directory so the replace is durable. A failure before the replace removes the staged file and leaves the + store untouched. The caller holds the store lock.""" + staged = path.with_name(f"{path.name}.tmp-{os.getpid()}-{uuid.uuid4().hex}") + try: + fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), 0o666) + try: + view = memoryview(data) + while view: + view = view[os.write(fd, view):] + os.fsync(fd) + finally: + os.close(fd) + os.replace(staged, path) + except BaseException: + try: + os.unlink(staged) + except OSError: + pass + raise + if sys.platform != "win32": + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +@contextmanager +def store_write_lock(path: Path, timeout: float = LOCK_TIMEOUT_SECONDS) -> Iterator[None]: + """Holds the exclusive store lock on `.lock`, the lock the C#, Rust and TypeScript runtimes take: flock on Unix, + LockFileEx over the whole file on Windows, polled every 10 ms with no handle held between polls. The lock file is + created if absent and never written or deleted.""" + lock_path = path.with_name(path.name + ".lock") + deadline = time.monotonic() + timeout + while True: + release = _try_lock(lock_path) + if release is not None: + break + if time.monotonic() >= deadline: + raise TimeoutError(f"CultCache timed out after {timeout} s waiting for the store lock {lock_path}.") + time.sleep(_LOCK_POLL_SECONDS) + try: + yield + finally: + release() + + +def _try_lock(lock_path: Path) -> Callable[[], None] | None: + if sys.platform == "win32": + return _try_lock_windows(lock_path) + import fcntl + + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o666) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + return None + return lambda: os.close(fd) # closing drops the flock + + +def _try_lock_windows(lock_path: Path) -> Callable[[], None] | None: + import ctypes + import msvcrt + from ctypes import wintypes + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p] + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_BINARY, 0o666) + handle = msvcrt.get_osfhandle(fd) + overlapped = ctypes.create_string_buffer(32) + # LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY over the whole range, as the TypeScript runtime takes it. + if not kernel.LockFileEx(handle, 0x3, 0, 0xFFFFFFFF, 0xFFFFFFFF, overlapped): + os.close(fd) + return None + return lambda: os.close(fd) # closing drops the lock + def _read_store(path: Path, read: Callable[[Path], _Contents]) -> _Contents | None: """Reads a store file, or returns None when nothing is at its path. @@ -351,7 +433,7 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None return decoded[0], envelopes -def _decode_legacy_envelopes(decoded: Any, msgpack: Any) -> list[CultCacheEnvelope]: +def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]: if not isinstance(decoded, list): raise ValueError("CultCache MessagePack store is not a recognized snapshot") envelopes: list[CultCacheEnvelope] = [] diff --git a/packages/cultcache-py/tests/test_slot_nil.py b/packages/cultcache-py/tests/test_slot_nil.py new file mode 100644 index 00000000..dca207bb --- /dev/null +++ b/packages/cultcache-py/tests/test_slot_nil.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +import msgpack + +from cultcache_py import database_entry_field, define_database_entry_type, define_record_codec, leaves as t + +# tests/vectors/slot-nil is written only by the C# reference (SlotNilVectorTests): what it reads and rewrites for a nil, a +# str in a byte[] slot, and a record short of its last slots. This runtime accepts what the reference accepts and rewrites +# the same bytes. +VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "slot-nil" + + +def _probe(): + inner = define_record_codec("SlotNilInner", [database_entry_field("x", 0, default="", codec=t.string)]) + return define_database_entry_type( + "tests.slot-nil", + [ + database_entry_field("s", 0, default="", codec=t.nullable(t.string)), + database_entry_field("i", 1, default=0, codec=t.int32), + database_entry_field("b", 2, default=b"", codec=t.nullable(t.bytes)), + database_entry_field("r", 3, default=None, codec=t.nullable(inner)), + database_entry_field("l", 4, default=[], codec=t.nullable(t.list(t.string))), + ], + ) + + +class SlotNilVectorTests(unittest.TestCase): + def test_slots_read_and_rewrite_exactly_what_the_reference_does_for_nil_str_in_bytes_and_short_records(self) -> None: + rows = msgpack.unpackb((VECTORS / "reads.msgpack").read_bytes(), raw=False) + probe = _probe() + self.assertEqual(17, len(rows)) + accepted = 0 + for name, payload, was_accepted, rewritten in rows: + with self.subTest(name): + if was_accepted: + self.assertEqual(rewritten.hex(), probe.encode_payload(probe.decode_payload(payload)).hex()) + accepted += 1 + else: + with self.assertRaises(Exception): + probe.decode_payload(payload) + self.assertEqual(15, accepted) + + def test_a_character_outside_the_bmp_is_four_utf8_bytes(self) -> None: + rows = {name: (payload, rewritten) for name, payload, _, rewritten in msgpack.unpackb((VECTORS / "reads.msgpack").read_bytes(), raw=False)} + probe = _probe() + self.assertEqual("\U0001F600", probe.decode_payload(rows["string-astral"][0])["s"]) + self.assertEqual(b"\xf0\x9f\x98\x80", probe.decode_payload(rows["bytes-str-astral"][0])["b"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-py/tests/test_store_lock.py b/packages/cultcache-py/tests/test_store_lock.py new file mode 100644 index 00000000..5ee29aa2 --- /dev/null +++ b/packages/cultcache-py/tests/test_store_lock.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +from cultcache_py import SingleFileMessagePackBackingStore, stores +from cultcache_py.backing_store import CultCacheEnvelope + +REPO = Path(__file__).resolve().parents[3] +TS_STORE_FILE = REPO / "packages" / "cultcache-ts" / "dist" / "store-file.js" +HOLD_SECONDS = 1.5 + +# A holder that takes the lock the way the Rust runtime does (an exclusive file lock), not through this package. +PY_HOLDER = """ +import fcntl, os, sys, time +fd = os.open(sys.argv[1] + '.lock', os.O_RDWR | os.O_CREAT, 0o666) +fcntl.flock(fd, fcntl.LOCK_EX) +print('HELD', flush=True) +time.sleep(float(sys.argv[2])) +""" +# A holder that is the TypeScript runtime's own withStoreWriteLock, built from packages/cultcache-ts. +TS_HOLDER = """ +const { withStoreWriteLock } = require(process.argv[1]); +withStoreWriteLock(process.argv[2], async () => { + console.log('HELD'); + await new Promise((r) => setTimeout(r, Number(process.argv[3]) * 1000)); +}); +""" + + +def envelope(key: str) -> CultCacheEnvelope: + return CultCacheEnvelope(key=key, type="t", payload=b"\x01", stored_at="2026-11-06T00:00:00Z") + + +def start_holder(argv: list[str]) -> subprocess.Popen[str]: + holder = subprocess.Popen(argv, stdout=subprocess.PIPE, text=True) + assert holder.stdout is not None and holder.stdout.readline().strip() == "HELD" + holder.held_at = time.monotonic() # type: ignore[attr-defined] + return holder + + +@unittest.skipIf(sys.platform == "win32", "the foreign holders here use flock; Windows is covered by the TS holder") +class StoreLockTests(unittest.TestCase): + def setUp(self) -> None: + self.dir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.path = self.dir / "state.cc" + + def assert_write_waits_for(self, holder: subprocess.Popen[str]) -> None: + self.addCleanup(holder.wait) + self.addCleanup(holder.stdout.close) # type: ignore[union-attr] + refused = SingleFileMessagePackBackingStore(self.path, lock_timeout=0.2) + with self.assertRaises(TimeoutError): + refused.push(envelope("blocked")) + self.assertFalse(self.path.exists(), "a write that could not take the lock must not touch the store") + + SingleFileMessagePackBackingStore(self.path, lock_timeout=10).push(envelope("after")) + self.assertGreaterEqual(time.monotonic() - holder.held_at, HOLD_SECONDS - 0.1, "the write finished while the holder still held the lock") # type: ignore[attr-defined] + self.assertEqual(["after"], [e.key for e in SingleFileMessagePackBackingStore(self.path).pull_all()]) + + def test_write_waits_while_a_foreign_process_holds_the_lock(self) -> None: + self.assert_write_waits_for(start_holder([sys.executable, "-c", PY_HOLDER, str(self.path), str(HOLD_SECONDS)])) + + @unittest.skipUnless(shutil.which("node") and TS_STORE_FILE.exists(), "needs node and a built packages/cultcache-ts (npm run build)") + def test_write_waits_while_the_typescript_runtime_holds_the_lock(self) -> None: + holder = start_holder(["node", "-e", TS_HOLDER, str(TS_STORE_FILE), str(self.path), str(HOLD_SECONDS)]) + self.assert_write_waits_for(holder) + + def test_the_lock_is_released_after_each_write(self) -> None: + store = SingleFileMessagePackBackingStore(self.path, lock_timeout=0.2) + for key in "abc": + store.push(envelope(key)) + self.assertEqual(["a", "b", "c"], sorted(e.key for e in store.pull_all())) + + def test_concurrent_processes_lose_no_writes(self) -> None: + script = ( + "import sys\n" + "from cultcache_py import SingleFileMessagePackBackingStore as S\n" + "from cultcache_py.backing_store import CultCacheEnvelope as E\n" + "s = S(sys.argv[1])\n" + "for i in range(25):\n" + " s.push(E(key=f'{sys.argv[2]}{i}', type='t', payload=b'\\x01', stored_at='x'))\n" + ) + env = {**os.environ, "PYTHONPATH": str(Path(stores.__file__).parents[1])} + procs = [subprocess.Popen([sys.executable, "-c", script, str(self.path), tag], env=env) for tag in "pqr"] + for proc in procs: + self.assertEqual(0, proc.wait()) + self.assertEqual(75, len(SingleFileMessagePackBackingStore(self.path).pull_all())) + + +class StagedWriteTests(unittest.TestCase): + def setUp(self) -> None: + self.dir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.path = self.dir / "state.cc" + + def test_the_staged_file_is_fsynced_before_the_replace(self) -> None: + order: list[str] = [] + real_fsync, real_replace = os.fsync, os.replace + with mock.patch("os.fsync", side_effect=lambda fd: (order.append("fsync"), real_fsync(fd))[1]), mock.patch( + "os.replace", side_effect=lambda a, b: (order.append("replace"), real_replace(a, b))[1] + ): + SingleFileMessagePackBackingStore(self.path).push(envelope("a")) + self.assertEqual("fsync", order[0]) + self.assertLess(order.index("fsync"), order.index("replace")) + + def test_the_staged_name_is_unique_per_write(self) -> None: + names: list[str] = [] + real_replace = os.replace + with mock.patch("os.replace", side_effect=lambda a, b: (names.append(os.fspath(a)), real_replace(a, b))[1]): + store = SingleFileMessagePackBackingStore(self.path) + store.push(envelope("a")) + store.push(envelope("b")) + self.assertEqual(2, len(set(names))) + self.assertNotIn(os.fspath(self.path) + ".tmp", names) + + def test_a_failed_replace_leaves_the_store_and_no_staged_file(self) -> None: + store = SingleFileMessagePackBackingStore(self.path) + store.push(envelope("a")) + before = self.path.read_bytes() + with mock.patch("os.replace", side_effect=OSError("boom")), self.assertRaises(OSError): + store.push(envelope("b")) + self.assertEqual(before, self.path.read_bytes()) + self.assertEqual([], [p.name for p in self.dir.iterdir() if ".tmp" in p.name]) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index f15fcee2..c417d535 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -13,6 +13,7 @@ only in git history. ### Added +- Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name or value find the one with the smallest key. A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`). - `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::()`. ## [0.3.1] diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index afae7285..2c5c8027 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -174,6 +174,9 @@ without pretending Rust has runtime reflection hiding under the floorboards. - `get_required_envelope::` - `get_all::` - `get_all_with_keys::` +- `get_by_name::` +- `get_by_index::` +- `get_global::` - `soa::` - `put::` - `put_envelope::` diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index 65e41b92..fe4e57c6 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -22,6 +22,7 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result = None; let mut schema_name: Option = None; + let mut global = false; for attribute in input .attrs @@ -37,6 +38,12 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result syn::Result ::std::option::Option<::std::string::String> { + ::cultcache_rs::CultLookupValue::lookup_value(&self.#member) + } + } + }); + let index_members = slots.iter().filter_map(|slot| slot.index.as_ref().map(|alias| (alias, &slot.ident))); + let index_pushes = index_members.map(|(alias, member)| { + quote! { + if let ::std::option::Option::Some(value) = ::cultcache_rs::CultLookupValue::lookup_value(&self.#member) { + held.push((#alias, value)); + } + } + }); + let index_pushes: Vec<_> = index_pushes.collect(); + let indexes_fn = (!index_pushes.is_empty()).then(|| { + quote! { + fn cult_indexes(&self) -> ::std::vec::Vec<(&'static str, ::std::string::String)> { + let mut held = ::std::vec::Vec::new(); + #(#index_pushes)* + held + } + } + }); Ok(quote! { impl ::cultcache_rs::DatabaseEntry for #ident { const TYPE: &'static str = #entry_type; const SCHEMA_NAME: &'static str = #schema_name; + const GLOBAL: bool = #global; + #name_fn + #indexes_fn } #codec @@ -133,6 +169,8 @@ fn named_slots(data: Data, ident: &syn::Ident, owner: &str) -> syn::Result = None; let mut default = false; let mut bytes = false; + let mut name = false; + let mut index: Option = None; for attribute in field .attrs .iter() @@ -157,6 +195,22 @@ fn named_slots(data: Data, ident: &syn::Ident, owner: &str) -> syn::Result()?.value() + } else { + String::new() + }; + index = Some(if alias.trim().is_empty() { field_ident.to_string() } else { alias }); + Ok(()) } else { Err(meta.error("unsupported field cultcache attribute")) } @@ -180,8 +234,28 @@ fn named_slots(data: Data, ident: &syn::Ident, owner: &str) -> syn::Result, } fn union_codec(input: DeriveInput) -> syn::Result { @@ -842,4 +920,89 @@ mod tests { let input: DeriveInput = syn::parse_str(source).expect("the fixture parses"); named_slots(input.data, &input.ident, "CultSlots").expect("the fixture is accepted"); } + fn entry_error(source: &str) -> String { + let input: DeriveInput = syn::parse_str(source).expect("the fixture parses"); + database_entry_codec(input).expect_err("the fixture is refused").to_string() + } + + fn entry_tokens(source: &str) -> String { + let input: DeriveInput = syn::parse_str(source).expect("the fixture parses"); + database_entry_codec(input).expect("the fixture is accepted").to_string().replace(' ', "") + } + + #[test] + fn name_index_and_global_generate_the_lookup_surface() { + let tokens = entry_tokens( + r#"#[cultcache(type = "t", global)] struct S { + #[cultcache(key = 0, name)] label: String, + #[cultcache(key = 1, index)] code: String, + #[cultcache(key = 2, index = "sku")] stock: Option, + }"#, + ); + assert!(tokens.contains("constGLOBAL:bool=true"), "{tokens}"); + assert!(tokens.contains("fncult_name"), "{tokens}"); + assert!(tokens.contains("(\"code\",value)"), "{tokens}"); + assert!(tokens.contains("(\"sku\",value)"), "{tokens}"); + } + + #[test] + fn a_record_without_lookups_generates_none_of_them() { + let tokens = entry_tokens(r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0)] a: String }"#); + assert!(tokens.contains("constGLOBAL:bool=false"), "{tokens}"); + assert!(!tokens.contains("fncult_name") && !tokens.contains("fncult_indexes"), "{tokens}"); + } + + // ResolveIndexAlias: a blank alias is the member's name. + #[test] + fn a_blank_index_alias_is_the_member_name() { + let tokens = entry_tokens( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = " ")] code: String }"#, + ); + assert!(tokens.contains("(\"code\",value)"), "{tokens}"); + } + + #[test] + fn two_names_are_refused_for_that_reason() { + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, name)] a: String, #[cultcache(key = 1, name)] b: String }"#, + ); + assert!(message.contains("more than one member"), "{message}"); + } + + // D10: two members that declare one alias; a bare index is the member name, so it collides with an explicit alias of it. + #[test] + fn two_members_with_one_index_alias_are_refused_for_that_reason() { + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = "k")] a: String, #[cultcache(key = 1, index = "k")] b: String }"#, + ); + assert!(message.contains("both declare index alias \"k\""), "{message}"); + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index)] a: String, #[cultcache(key = 1, index = "a")] b: String }"#, + ); + assert!(message.contains("both declare index alias \"a\""), "{message}"); + } + + #[test] + fn a_repeated_lookup_attribute_is_refused() { + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, name, name)] a: String }"#, + ); + assert!(message.contains("repeated `name`"), "{message}"); + let message = entry_error( + r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index, index = "x")] a: String }"#, + ); + assert!(message.contains("repeated `index`"), "{message}"); + let message = entry_error(r#"#[cultcache(type = "t", global, global)] struct S { #[cultcache(key = 0)] a: String }"#); + assert!(message.contains("repeated `global`"), "{message}"); + } + + // Only a DatabaseEntry is looked up: a nested slot record carrying the attributes is refused as before. + #[test] + fn lookup_attributes_are_refused_on_a_slots_record() { + let message = slots_error(r#"struct S { #[cultcache(key = 0, name)] a: String }"#); + assert!(message.contains("unsupported field cultcache attribute"), "{message}"); + let message = slots_error(r#"struct S { #[cultcache(key = 0, index)] a: String }"#); + assert!(message.contains("unsupported field cultcache attribute"), "{message}"); + } + } diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index b77f4971..4ceb43f2 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -162,8 +162,68 @@ pub use cultcache_rs_derive::DatabaseEntry; pub trait DatabaseEntry: Serialize + DeserializeOwned + Clone + Send + 'static { const TYPE: &'static str; const SCHEMA_NAME: &'static str = "DatabaseEntry"; + /// `#[cultcache(global)]`: the cache holds at most one record of this type, found by `get_global`. + const GLOBAL: bool = false; + /// The `#[cultcache(name)]` member's value, which `get_by_name` finds the record by. `None` is no name. + fn cult_name(&self) -> Option { + None + } + /// Each `#[cultcache(index)]` member as (alias, value), which `get_by_index` finds the record by. + fn cult_indexes(&self) -> Vec<(&'static str, String)> { + Vec::new() + } } +/// What a `#[cultcache(name)]` or `#[cultcache(index)]` member says it is called: its text, or `None` for a member +/// that holds no value (an `Option` that is `None`). The cache treats an empty name and a blank index value as none. +pub trait CultLookupValue { + fn lookup_value(&self) -> Option; +} + +impl CultLookupValue for String { + fn lookup_value(&self) -> Option { + Some(self.clone()) + } +} + +impl CultLookupValue for Option { + fn lookup_value(&self) -> Option { + self.as_ref().and_then(CultLookupValue::lookup_value) + } +} + +macro_rules! lookup_value_by_display { + ($($ty:ty),* $(,)?) => {$( + impl CultLookupValue for $ty { + fn lookup_value(&self) -> Option { + Some(self.to_string()) + } + } + )*}; +} + +lookup_value_by_display!(bool, char, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); + +/// A global type already holds a record under another key; a second would leave `get_global` with two answers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CultCacheSecondGlobal { + pub schema_name: String, + pub existing_key: String, + pub key: String, +} + +impl std::fmt::Display for CultCacheSecondGlobal { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "{} is a global and already has record {}; {} would be a second.", + self.schema_name, self.existing_key, self.key + ) + } +} + +impl std::error::Error for CultCacheSecondGlobal {} + /// A type written as a positional slots array: what the `CultSlots` and `DatabaseEntry` /// derives implement, and what a `CultUnion` arm must carry. #[diagnostic::on_unimplemented( @@ -2655,6 +2715,8 @@ struct CultCacheStoreRegistration { pub struct CultCache { definitions: BTreeMap, + /// The registered types declared `#[cultcache(global)]`. + globals: BTreeSet, schema_name_definitions: BTreeMap, entries: BTreeMap<(String, String), CultCacheEnvelope>, stores: Vec, @@ -2664,6 +2726,7 @@ impl CultCache { pub fn new() -> Self { Self { definitions: BTreeMap::new(), + globals: BTreeSet::new(), schema_name_definitions: BTreeMap::new(), entries: BTreeMap::new(), stores: Vec::new(), @@ -2694,6 +2757,11 @@ impl CultCache { )); } self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME); + if T::GLOBAL { + self.globals.insert(T::TYPE.to_string()); + } else { + self.globals.remove(T::TYPE); + } self.schema_name_definitions .insert(T::SCHEMA_NAME.to_string(), T::TYPE.to_string()); Ok(()) @@ -2800,6 +2868,7 @@ impl CultCache { loaded.insert(entry_id(&entry), entry); } } + self.refuse_second_globals(&loaded)?; self.entries = loaded; Ok(()) } @@ -2883,6 +2952,65 @@ impl CultCache { Ok(values) } + /// The one record of a `#[cultcache(global)]` type, or `None` before it is written (and for a type that is + /// not a global). + pub fn get_global(&self) -> Result> { + self.require_entry_type::()?; + if !T::GLOBAL { + return Ok(None); + } + self.find_record::(|_| true) + } + + /// The record whose `#[cultcache(name)]` member is `name`. An empty name is no name, so it finds nothing. Records + /// of one type sharing a name find the one with the smallest key (ordinal), whatever order they were written or + /// loaded in; removing it finds the next. + pub fn get_by_name(&self, name: &str) -> Result> { + self.require_entry_type::()?; + if name.is_empty() { + return Ok(None); + } + self.find_record::(|document| document.cult_name().as_deref() == Some(name)) + } + + /// The record whose `#[cultcache(index)]` member `alias` holds `value`. A blank value is not indexed, so it finds + /// nothing. Records of one type sharing a value find the one with the smallest key (ordinal), whatever order they + /// were written or loaded in; removing it finds the next. + pub fn get_by_index(&self, alias: &str, value: &str) -> Result> { + self.require_entry_type::()?; + if value.trim().is_empty() { + return Ok(None); + } + self.find_record::(|document| { + document + .cult_indexes() + .iter() + .any(|(held_alias, held_value)| *held_alias == alias && held_value == value) + }) + } + + /// The first record of `T`, in key order, that `matches`. + fn find_record(&self, matches: impl Fn(&T) -> bool) -> Result> { + let held = self + .entries + .range((T::TYPE.to_string(), String::new())..) + .take_while(|((held_type, _), _)| held_type == T::TYPE); + for (_, entry) in held { + let document: T = decode_payload(&entry.payload).with_context(|| { + format!( + "failed to decode CultCache entry {:?} at key {:?} as {}", + T::TYPE, + entry.key, + T::SCHEMA_NAME + ) + })?; + if matches(&document) { + return Ok(Some(document)); + } + } + Ok(None) + } + pub fn soa(&self) -> Result> { let keyed_rows = self.get_all_with_keys::()?; let (keys, documents): (Vec<_>, Vec<_>) = keyed_rows.into_iter().unzip(); @@ -2900,6 +3028,7 @@ impl CultCache { pub fn put(&mut self, key: impl Into, value: &T) -> Result { let (entry, parsed) = self.prepare_entry(key, value)?; + self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.push(&entry)?; } @@ -3011,6 +3140,7 @@ impl CultCache { for entry in &staged { next.insert(entry_id(entry), entry.clone()); } + self.refuse_second_globals(&next)?; if let Some(store_index) = store_index { // The home store receives its own records only: those it already // holds in this cache plus the staged ones. @@ -3029,6 +3159,7 @@ impl CultCache { pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { let parsed = self.validate_envelope::(&entry)?; + self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.push(&entry)?; } @@ -3061,6 +3192,7 @@ impl CultCache { )); } require_payload_depth(&entry)?; + self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; if let Some(home) = self.home_index(&entry.r#type)? { self.stores[home].store.push(&entry)?; } @@ -3146,10 +3278,43 @@ impl CultCache { self.home_index(&entry.r#type)?; loaded.insert(entry_id(&entry), entry); } + self.refuse_second_globals(&loaded)?; self.entries = loaded; Ok(()) } + fn refuse_second_global( + &self, + entries: &BTreeMap<(String, String), CultCacheEnvelope>, + type_id: &str, + key: &str, + ) -> Result<()> { + if !self.globals.contains(type_id) { + return Ok(()); + } + let held_other = entries + .range((type_id.to_string(), String::new())..) + .take_while(|((held_type, _), _)| held_type == type_id) + .map(|((_, held_key), _)| held_key) + .find(|held_key| held_key.as_str() != key); + match held_other { + Some(existing_key) => Err(CultCacheSecondGlobal { + schema_name: self.definitions.get(type_id).copied().unwrap_or(type_id).to_string(), + existing_key: existing_key.clone(), + key: key.to_string(), + } + .into()), + None => Ok(()), + } + } + + fn refuse_second_globals(&self, entries: &BTreeMap<(String, String), CultCacheEnvelope>) -> Result<()> { + for (type_id, key) in entries.keys() { + self.refuse_second_global(entries, type_id, key)?; + } + Ok(()) + } + fn require_entry_type(&self) -> Result<()> { match self.definitions.get(T::TYPE) { Some(schema_name) if *schema_name == T::SCHEMA_NAME => Ok(()), diff --git a/packages/cultcache-rs/tests/lookups.rs b/packages/cultcache-rs/tests/lookups.rs new file mode 100644 index 00000000..2467fbc6 --- /dev/null +++ b/packages/cultcache-rs/tests/lookups.rs @@ -0,0 +1,323 @@ +//! Lookups parity with the C# reference: `GetByName`, `GetByIndex` and `GetGlobal` +//! (`src/GameCult.Caching/CultCache.cs`) and the one-global rule its commit enforces. + +use cultcache_rs::{ + CacheBackingStore, CultCache, CultCacheSecondGlobal, DatabaseEntry, SingleFileMessagePackBackingStore, +}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "item")] +struct Item { + #[cultcache(key = 0, name)] + name: String, + #[cultcache(key = 1, index)] + category: String, + #[cultcache(key = 2, index = "sku")] + code: Option, + #[cultcache(key = 3, default)] + value: u32, +} + +fn item(name: &str, category: &str, code: Option<&str>) -> Item { + Item { name: name.into(), category: category.into(), code: code.map(Into::into), value: 0 } +} + +/// Same names and values as `Item`, a different type: lookups never cross types. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "gadget")] +struct Gadget { + #[cultcache(key = 0, name)] + name: String, + #[cultcache(key = 1, index)] + category: String, +} + +/// Sorts after `Item` and has its exact shape: a lookup that ran past the end of its own type would answer from it. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "zed")] +struct Zed { + #[cultcache(key = 0, name)] + name: String, + #[cultcache(key = 1, index)] + category: String, + #[cultcache(key = 2, index = "sku")] + code: Option, + #[cultcache(key = 3, default)] + value: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "settings", global)] +struct Settings { + #[cultcache(key = 0)] + value: String, +} + +fn settings(value: &str) -> Settings { + Settings { value: value.into() } +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "plain")] +struct Plain { + #[cultcache(key = 0)] + value: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "numbered")] +struct Numbered { + #[cultcache(key = 0, name)] + id: u32, + #[cultcache(key = 1, index)] + owner: u64, +} + +fn cache() -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache +} + +fn reopened(path: &std::path::Path) -> CultCache { + let mut reopened = cache(); + reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap(); + reopened.pull_all_backing_stores().unwrap(); + reopened +} + +fn stored_cache(path: &std::path::Path) -> CultCache { + let mut stored = cache(); + stored.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap(); + stored.pull_all_backing_stores().unwrap(); + stored +} + +// The C# FirstCall snippet: a record written to a .cc file is found by its [CultName] through a second cache. +#[test] +fn a_record_is_found_by_name_and_by_index_after_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lookups.cc"); + let mut written = stored_cache(&path); + written.put("item:potion", &Item { value: 50, ..item("Potion", "Consumable", Some("P-1")) }).unwrap(); + + let reopened = reopened(&path); + let by_name = reopened.get_by_name::("Potion").unwrap().unwrap(); + assert_eq!(by_name.value, 50); + assert_eq!(reopened.get_by_index::("category", "Consumable").unwrap(), Some(by_name.clone())); + assert_eq!(reopened.get_by_index::("sku", "P-1").unwrap(), Some(by_name)); +} + +#[test] +fn a_missing_name_or_value_is_none_and_an_unregistered_type_is_refused() { + let mut cache = cache(); + cache.put("item:a", &item("A", "cat", Some("a-1"))).unwrap(); + + assert_eq!(cache.get_by_name::("B").unwrap(), None); + assert_eq!(cache.get_by_name::("a").unwrap(), None, "names compare ordinally"); + assert_eq!(cache.get_by_index::("category", "other").unwrap(), None); + assert_eq!(cache.get_by_index::("nosuchalias", "cat").unwrap(), None); + assert_eq!(cache.get_by_index::("code", "a-1").unwrap(), None, "the alias is sku, not the member name"); + + let bare = CultCache::new(); + assert!(bare.get_by_name::("A").is_err()); + assert!(bare.get_by_index::("category", "cat").is_err()); + assert!(bare.get_global::().is_err()); +} + +#[test] +fn an_index_without_an_alias_is_named_by_its_member() { + let mut cache = cache(); + cache.put("item:a", &item("A", "cat", None)).unwrap(); + assert!(cache.get_by_index::("category", "cat").unwrap().is_some()); +} + +// A blank name is no name; a blank index value is not indexed; a nil member holds no value. +#[test] +fn blank_and_nil_values_are_not_found() { + let mut cache = cache(); + cache.put("item:blank", &item("", " ", None)).unwrap(); + assert_eq!(cache.get_by_name::("").unwrap(), None); + assert_eq!(cache.get_by_index::("category", " ").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", "").unwrap(), None); + + // Only emptiness voids a name: whitespace is a name, as in C# (Length > 0). + cache.put("item:space", &item(" ", "x", None)).unwrap(); + assert_eq!(cache.get_by_name::(" ").unwrap().unwrap().category, "x"); +} + +#[test] +fn lookups_never_cross_types() { + let mut cache = cache(); + cache.put("item:a", &item("same", "same", None)).unwrap(); + cache.put("gadget:a", &Gadget { name: "same".into(), category: "same".into() }).unwrap(); + cache.put("zed:a", &Zed { name: "same".into(), category: "same".into(), code: None, value: 0 }).unwrap(); + + assert_eq!(cache.get_by_name::("same").unwrap().unwrap().name, "same"); + cache.delete::("item:a").unwrap(); + assert_eq!(cache.get_by_name::("same").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "same").unwrap(), None); + assert!(cache.get_by_name::("same").unwrap().is_some()); + assert!(cache.get_by_index::("category", "same").unwrap().is_some()); +} + +#[test] +fn a_name_and_index_follow_a_rewritten_record_and_leave_a_removed_one() { + let mut cache = cache(); + cache.put("item:a", &item("old", "c1", None)).unwrap(); + cache.put("item:a", &item("new", "c2", None)).unwrap(); + + assert_eq!(cache.get_by_name::("old").unwrap(), None, "a stale name is not found"); + assert_eq!(cache.get_by_index::("category", "c1").unwrap(), None, "a stale value is not found"); + assert!(cache.get_by_name::("new").unwrap().is_some()); + assert!(cache.get_by_index::("category", "c2").unwrap().is_some()); + + cache.delete::("item:a").unwrap(); + assert_eq!(cache.get_by_name::("new").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "c2").unwrap(), None); +} + +#[test] +fn non_string_members_are_found_by_their_text() { + let mut cache = cache(); + let owner = 161_u64; + cache.put("numbered:1", &Numbered { id: 42, owner }).unwrap(); + assert!(cache.get_by_name::("42").unwrap().is_some()); + assert!(cache.get_by_index::("owner", "161").unwrap().is_some()); +} + +// VariantTests.RemovingTheHolderADuplicateValueFindsLeavesTheOtherFindable: records sharing a value find the one with +// the smallest key, whatever the write order, live and after a reopen; removing the holder finds the other. +#[test] +fn records_sharing_a_value_find_the_smallest_key_and_removing_it_finds_the_next() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("shared.cc"); + { + let mut cache = stored_cache(&path); + cache.put("item:p2", &item("p2", "x", None)).unwrap(); + cache.put("item:p1", &item("p1", "x", None)).unwrap(); + let found = |cache: &CultCache| cache.get_by_index::("category", "x").unwrap().map(|found| found.name); + assert_eq!(found(&cache).as_deref(), Some("p1"), "the smallest key, though written second"); + assert_eq!(reopened(&path).get_by_index::("category", "x").unwrap().unwrap().name, "p1"); + + cache.delete::("item:p1").unwrap(); + assert_eq!(found(&cache).as_deref(), Some("p2"), "removing the holder finds the other one"); + + cache.put("item:p1", &item("p1", "x", None)).unwrap(); + assert_eq!(found(&cache).as_deref(), Some("p1")); + cache.delete::("item:p2").unwrap(); + assert_eq!(found(&cache).as_deref(), Some("p1"), "removing the other changes nothing"); + cache.delete::("item:p1").unwrap(); + assert_eq!(found(&cache), None); + } + assert_eq!(reopened(&path).get_by_index::("category", "x").unwrap(), None); +} + +#[test] +fn records_sharing_a_name_find_the_smallest_key() { + let mut cache = cache(); + cache.put("item:b", &Item { value: 2, ..item("twin", "c", None) }).unwrap(); + cache.put("item:a", &Item { value: 1, ..item("twin", "c", None) }).unwrap(); + assert_eq!(cache.get_by_name::("twin").unwrap().unwrap().value, 1); + cache.delete::("item:a").unwrap(); + assert_eq!(cache.get_by_name::("twin").unwrap().unwrap().value, 2); +} + +// BackingStoreTests.HydratesPersistedGlobal: opening a store invents no global; a written one survives a reopen. +#[test] +fn a_global_is_none_until_written_and_survives_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("global.cc"); + { + let mut seed = stored_cache(&path); + assert_eq!(seed.get_global::().unwrap(), None, "opening a store invented a global"); + seed.put("global:settings", &settings("durable authority")).unwrap(); + assert_eq!(seed.get_global::().unwrap(), Some(settings("durable authority"))); + } + assert_eq!(reopened(&path).get_global::().unwrap(), Some(settings("durable authority"))); +} + +#[test] +fn a_type_that_is_not_a_global_has_no_global() { + let mut cache = cache(); + cache.put("plain:one", &Plain { value: "v".into() }).unwrap(); + assert_eq!(cache.get_global::().unwrap(), None); +} + +#[test] +fn a_second_global_under_another_key_is_refused_and_writes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("second.cc"); + let mut cache = stored_cache(&path); + cache.put("g1", &settings("one")).unwrap(); + + let refusal = cache.put("g2", &settings("two")).unwrap_err(); + let second = refusal.downcast_ref::().expect("a typed refusal"); + assert_eq!((second.existing_key.as_str(), second.key.as_str()), ("g1", "g2")); + assert!(refusal.to_string().contains("is a global and already has record g1; g2 would be a second.")); + assert_eq!(cache.get_global::().unwrap(), Some(settings("one"))); + assert_eq!(reopened(&path).get_global::().unwrap(), Some(settings("one")), "nothing reached the store"); + + // The same key is the same record, not a second. + cache.put("g1", &settings("one again")).unwrap(); + assert_eq!(cache.get_global::().unwrap(), Some(settings("one again"))); +} + +// RetypeEvictionTests.EvictingAGlobalFreesItsSlot: removing the global frees the slot for another key. +#[test] +fn removing_a_global_frees_its_slot() { + let mut cache = cache(); + cache.put("g1", &settings("one")).unwrap(); + assert!(cache.put("g2", &settings("two")).is_err()); + cache.delete::("g1").unwrap(); + assert_eq!(cache.get_global::().unwrap(), None); + cache.put("g2", &settings("two")).unwrap(); + assert_eq!(cache.get_global::().unwrap(), Some(settings("two"))); +} + +#[test] +fn every_write_path_refuses_a_second_global() { + let mut cache = cache(); + cache.put("g1", &settings("one")).unwrap(); + let (second, _) = cache.prepare_entry("g2", &settings("two")).unwrap(); + + assert!(cache.put_envelope::(second.clone()).unwrap_err().is::()); + assert!(cache.put_raw_envelope(second.clone()).unwrap_err().is::()); + let (plain, _) = cache.prepare_entry("plain:1", &Plain { value: "v".into() }).unwrap(); + assert!(cache.put_prepared_batch(vec![plain, second]).unwrap_err().is::()); + assert_eq!(cache.get::("plain:1").unwrap(), None, "a refused batch lands none of it"); + assert_eq!(cache.get_global::().unwrap(), Some(settings("one"))); +} + +#[test] +fn a_batch_that_stages_two_globals_is_refused_whole() { + let mut cache = cache(); + let (one, _) = cache.prepare_entry("g1", &settings("one")).unwrap(); + let (two, _) = cache.prepare_entry("g2", &settings("two")).unwrap(); + assert!(cache.put_prepared_batch(vec![one, two]).unwrap_err().is::()); + assert_eq!(cache.get_global::().unwrap(), None); +} + +// StoreRoutingTests: a store holding two globals is refused on load rather than answering one of them. +#[test] +fn loading_a_store_that_holds_two_globals_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("two.cc"); + let source = cache(); + let mut store = SingleFileMessagePackBackingStore::new(&path); + for (key, value) in [("g1", "one"), ("g2", "two")] { + let (entry, _) = source.prepare_entry(key, &settings(value)).unwrap(); + store.push(&entry).unwrap(); + } + + let mut loading = cache(); + loading.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + assert!(loading.pull_all_backing_stores().unwrap_err().is::()); + assert_eq!(loading.get_global::().unwrap(), None, "a refused load installs nothing"); +} diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index a2f10c54..93e17239 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -15,6 +15,8 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following ### Added +- `@gamecult/cultcache-ts/browser`, also what a bundler resolves for the package root under the `browser` export condition: the root surface without `SingleFileMessagePackBackingStore` and `readSingleFileStore`, so it imports no `node:*` built-in and no `koffi` and bundles with esbuild `platform: "browser"`. The Node entry exports exactly what it did. + - `readSingleFileStore(path)`, the one synchronous reader of a single-file store, for consumers that must read a store file rather than decode it themselves. `pullAll`, `push`, `delete` and `pushAll` ask the same reader. - `StoreUnreadableError` for a store file this runtime cannot read, `SchemaConflictError` for a write the catalog cannot describe, and `schemaIdentityOf` with its result type `CultCacheSchemaIdentity` (`schemaId`, `schemaName`, `schemaVersion`, `compatibleSchemaIds`), the one derivation of a definition's identity. - `SingleFileStoreRead`, exported: the result type of `readSingleFileStore` (`absent`, `format`, `envelopes`, `repairedLegacyPayload`). diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index 192d4b5d..b3a85345 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -61,7 +61,7 @@ const note = decodeDocumentPayload(noteDocument, envelope.payload); // parsed No const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload ``` -Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. +Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. The whole cache without the single-file store is `@gamecult/cultcache-ts/browser`, which is also what bundlers resolve for the package root (`browser` condition); bring your own `CacheBackingStore`. ## `.cc` Inspection diff --git a/packages/cultcache-ts/package.json b/packages/cultcache-ts/package.json index c48e54e5..ffe1463d 100644 --- a/packages/cultcache-ts/package.json +++ b/packages/cultcache-ts/package.json @@ -7,9 +7,15 @@ "exports": { ".": { "types": "./dist/index.d.ts", + "browser": "./dist/browser.js", "require": "./dist/index.js", "default": "./dist/index.js" }, + "./browser": { + "types": "./dist/browser.d.ts", + "require": "./dist/browser.js", + "default": "./dist/browser.js" + }, "./document": { "types": "./dist/document.d.ts", "require": "./dist/document.js", @@ -25,6 +31,9 @@ "*": { "document": [ "dist/document.d.ts" + ], + "browser": [ + "dist/browser.d.ts" ] } }, diff --git a/packages/cultcache-ts/src/browser.ts b/packages/cultcache-ts/src/browser.ts new file mode 100644 index 00000000..bf62976f --- /dev/null +++ b/packages/cultcache-ts/src/browser.ts @@ -0,0 +1,49 @@ +// Browser-safe surface: everything the package root exports except the node-only single-file store +// (node:fs, koffi). Bring your own CacheBackingStore, or none. +export * as codecs from "./codec"; +export type { Codec, SlotsCodec } from "./codec"; +export { CultCache, CultCacheBuilder } from "./cult-cache"; +export { inspectCultCacheBytes } from "./cult-cache-inspector"; +export { + decodeDocumentPayload, + defineDocumentRegistry, + defineDocumentType, + documentPayloadBytes, + documentPayloadFromBytes, + encodeDocumentPayload, + schemaIdentityOf, + type CultCacheSchemaIdentity, +} from "./document"; +export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; +export type { DocumentPayload } from "./document"; +export { SchemaConflictError, StoreUnreadableError } from "./store-format"; +export type { + AnyCultCacheDocumentDefinition, + CacheBackingStore, + CultCacheDocumentAccessor, + CultCacheDocumentDefinition, + CultCacheDocumentFieldName, + CultCacheDocumentFormatter, + CultCacheDocumentIndexDefinition, + CultCacheDocumentRegistry, + CultCacheSchemaCatalogEntry, + CultCacheSchemaCatalogMember, + CultCacheSchema, + CultCacheDocumentValue, + CultCacheEnvelope, + CultCacheIndexScalar, + InferCultCacheSchemaValue, + CultCacheStoreRegistration, + PushAllOptions, +} from "./types"; +export type { + CultCacheInspection, + InspectedCatalogEntry, + InspectedCatalogMember, + InspectedRecord, +} from "./cult-cache-inspector"; +export { + defineSwarmDocuments, + type DefineDocumentType, + type SwarmDocumentCatalog, +} from "./swarm-documents"; diff --git a/packages/cultcache-ts/src/index.ts b/packages/cultcache-ts/src/index.ts index 1facf6e2..f55dd049 100644 --- a/packages/cultcache-ts/src/index.ts +++ b/packages/cultcache-ts/src/index.ts @@ -1,48 +1,2 @@ -export * as codecs from "./codec"; -export type { Codec, SlotsCodec } from "./codec"; -export { CultCache, CultCacheBuilder } from "./cult-cache"; -export { inspectCultCacheBytes } from "./cult-cache-inspector"; -export { - decodeDocumentPayload, - defineDocumentRegistry, - defineDocumentType, - documentPayloadBytes, - documentPayloadFromBytes, - encodeDocumentPayload, - schemaIdentityOf, - type CultCacheSchemaIdentity, -} from "./document"; -export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth"; -export type { DocumentPayload } from "./document"; +export * from "./browser"; export { SingleFileMessagePackBackingStore, readSingleFileStore, type SingleFileStoreRead } from "./single-file-messagepack-backing-store"; -export { SchemaConflictError, StoreUnreadableError } from "./store-format"; -export type { - AnyCultCacheDocumentDefinition, - CacheBackingStore, - CultCacheDocumentAccessor, - CultCacheDocumentDefinition, - CultCacheDocumentFieldName, - CultCacheDocumentFormatter, - CultCacheDocumentIndexDefinition, - CultCacheDocumentRegistry, - CultCacheSchemaCatalogEntry, - CultCacheSchemaCatalogMember, - CultCacheSchema, - CultCacheDocumentValue, - CultCacheEnvelope, - CultCacheIndexScalar, - InferCultCacheSchemaValue, - CultCacheStoreRegistration, - PushAllOptions, -} from "./types"; -export type { - CultCacheInspection, - InspectedCatalogEntry, - InspectedCatalogMember, - InspectedRecord, -} from "./cult-cache-inspector"; -export { - defineSwarmDocuments, - type DefineDocumentType, - type SwarmDocumentCatalog, -} from "./swarm-documents"; diff --git a/packages/cultcache-ts/test/browser-entry.test.ts b/packages/cultcache-ts/test/browser-entry.test.ts new file mode 100644 index 00000000..bbf2e2c4 --- /dev/null +++ b/packages/cultcache-ts/test/browser-entry.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { join } from "node:path"; +import test from "node:test"; +import { runInNewContext } from "node:vm"; +import { build } from "esbuild"; + +// dist-test/test/ -> dist-test/src/ +const srcDir = join(__dirname, "..", "src"); + +async function bundleForBrowser(entry: string): Promise { + const result = await build({ + entryPoints: [join(srcDir, entry)], + bundle: true, + platform: "browser", + format: "iife", + globalName: "cultcache", + write: false, + logLevel: "silent", + }); + return result.outputFiles[0].text; +} + +test("the browser entry bundles for platform=browser and runs without node", async () => { + const code = await bundleForBrowser("browser.js"); + assert.ok(!/node:fs|koffi/.test(code), "no node-only store in the browser bundle"); + // A context with no require/process/Buffer: only what a browser page has. + const sandbox: Record = { TextEncoder, TextDecoder }; + runInNewContext(code + "\nglobalThis.__api = cultcache;", sandbox); + const api = sandbox.__api as Record; + assert.equal(typeof api.CultCache, "function"); + assert.equal(typeof api.defineDocumentType, "function"); + assert.equal(api.SingleFileMessagePackBackingStore, undefined); +}); + +test("the node entry still exports the single-file store and everything the browser entry has", async () => { + const node = await import(join(srcDir, "index.js")); + const browser = await import(join(srcDir, "browser.js")); + assert.equal(typeof node.SingleFileMessagePackBackingStore, "function"); + assert.equal(typeof node.readSingleFileStore, "function"); + for (const name of Object.keys(browser)) assert.ok(name in node, `node entry lost ${name}`); +}); + +test("the package root keeps the node-only store out of the browser build", async () => { + await assert.rejects(bundleForBrowser("index.js"), /node:fs|node:path|koffi/); +}); diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index 3fc36683..bbcc94f2 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -38,6 +38,7 @@ from .cultmesh_contracts import ( ) from .framing import decode_frame, encode_frame, read_frame, write_frame from .messages import ( + CULTNET_ERROR_CODES, CultNetMessage, CultNetWitnessArtifactBundle, compute_simulation_claim_hash, @@ -47,6 +48,7 @@ from .messages import ( document_delete, document_put_raw, encode_witness_artifact_bundle_payload, + error_message, hello, login, login_success, @@ -165,6 +167,7 @@ __all__ = [ "simulation_fact_document", "verse_catalog_request", "verse_from_wire", + "CULTNET_ERROR_CODES", "CultNetMessage", "CultNetWitnessArtifactBundle", "CultNetDatabaseSubscription", @@ -247,6 +250,7 @@ __all__ = [ "encode_witness_artifact_bundle_payload", "encode_frame", "encode_rudp_packet", + "error_message", "hello", "INTEROP_WIRE_CONTRACT", "login", diff --git a/packages/cultnet-py/src/cultnet_py/messages.py b/packages/cultnet-py/src/cultnet_py/messages.py index 4fa8d336..a5c0887e 100644 --- a/packages/cultnet-py/src/cultnet_py/messages.py +++ b/packages/cultnet-py/src/cultnet_py/messages.py @@ -28,9 +28,61 @@ def parse_message(payload: bytes) -> CultNetMessage: raise ValueError("CultNet schema-v0 messages must declare schemaVersion") body = dict(decoded) del body["schemaVersion"] + if schema_version == "cultnet.error.v0": + _validate_error_body(body) return CultNetMessage(schema_version=schema_version, body=body) +# Closed refusal-code set of cultnet.error.v0 (contracts/cultnet/cultnet.error.schema.json). Adding a +# code is one coordinated change across every runtime; a peer refuses a code it does not know. +CULTNET_ERROR_CODES: tuple[str, ...] = ( + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", +) + +_ERROR_DETAILS_KEYS = ("field", "value", "asOf", "current") + + +def _is_uint(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _validate_error_body(body: dict[str, Any]) -> None: + """Refuse a cultnet.error.v0 body that is not the full-key shape the schema requires.""" + expected = ("error", "routingHint", "code", "details") + if set(body) != set(expected): + raise ValueError(f"cultnet.error.v0 must carry exactly {list(expected)}, got {sorted(body)}") + if not isinstance(body["error"], str) or not body["error"]: + raise ValueError("cultnet.error.v0 error must be a non-empty string") + code = body["code"] + if code is not None and (not isinstance(code, str) or code not in CULTNET_ERROR_CODES): + raise ValueError(f"cultnet.error.v0 code {code!r} is not a recognized refusal code") + hint = body["routingHint"] + if hint is not None: + if ( + not isinstance(hint, dict) + or set(hint) != {"shard", "reason"} + or not isinstance(hint["reason"], str) + or not (hint["shard"] is None or isinstance(hint["shard"], dict)) + ): + raise ValueError("cultnet.error.v0 routingHint must be null or {shard, reason}") + details = body["details"] + if details is not None: + if not isinstance(details, dict) or set(details) != set(_ERROR_DETAILS_KEYS): + raise ValueError(f"cultnet.error.v0 details must be null or carry exactly {list(_ERROR_DETAILS_KEYS)}") + for key in ("field", "value"): + if details[key] is not None and not isinstance(details[key], str): + raise ValueError(f"cultnet.error.v0 details.{key} must be a string or null") + for key in ("asOf", "current"): + if details[key] is not None and not _is_uint(details[key]): + raise ValueError(f"cultnet.error.v0 details.{key} must be a non-negative integer or null") + + def hello( *, runtime_id: str, @@ -99,6 +151,29 @@ def verify_session( ) +def error_message( + error: str, + *, + code: str | None = None, + details: dict[str, Any] | None = None, + routing_hint: dict[str, Any] | None = None, +) -> CultNetMessage: + """Build cultnet.error.v0 with every key written (nil included), as the C# map formatter does. + + ``details`` may name any of field/value/asOf/current; the rest are written as null. + """ + body = { + "error": error, + "routingHint": routing_hint, + "code": code, + "details": None if details is None else {key: details.get(key) for key in _ERROR_DETAILS_KEYS}, + } + if details is not None and set(details) - set(_ERROR_DETAILS_KEYS): + raise ValueError(f"cultnet.error.v0 details has unknown keys {sorted(set(details) - set(_ERROR_DETAILS_KEYS))}") + _validate_error_body(body) + return CultNetMessage("cultnet.error.v0", body) + + def login_success( *, nonce: str, diff --git a/packages/cultnet-py/src/cultnet_py/schema_catalog.py b/packages/cultnet-py/src/cultnet_py/schema_catalog.py index 6d71b3cd..908a777f 100644 --- a/packages/cultnet-py/src/cultnet_py/schema_catalog.py +++ b/packages/cultnet-py/src/cultnet_py/schema_catalog.py @@ -12,6 +12,17 @@ PEER_EXCHANGE_REQUEST = "cultmesh.peer_exchange_request.v0" TRANSPORT_PROFILE_SCHEMA_VERSION = "cultnet.transport_profile.v0" TRANSPORT_PROFILE_SCHEMA_ID = f"{CULTNET_SCHEMA_BASE}/cultnet.transport-profile.schema.json" +# The closed code set of contracts/cultnet/cultnet.error.schema.json; a code outside it is one coordinated change across every runtime. +CULTNET_ERROR_CODES = ( + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", +) + WIRE_MESSAGE_SCHEMA_VERSIONS = ( ("cultnet.hello.v0", "CultNet Hello Message", f"{CULTNET_SCHEMA_BASE}/cultnet.hello.schema.json"), ("cultnet.error.v0", "CultNet Error Message", f"{CULTNET_SCHEMA_BASE}/cultnet.error.schema.json"), @@ -269,7 +280,7 @@ def shared_contract_schema_json(schema_id: str, title: str, schema_version: str) def wire_message_required_fields(schema_version: str) -> list[str]: required = { "cultnet.hello.v0": ["schemaVersion", "runtimeId"], - "cultnet.error.v0": ["schemaVersion", "error"], + "cultnet.error.v0": ["schemaVersion", "error", "routingHint", "code", "details"], "cultnet.login.v0": ["schemaVersion", "nonce", "auth", "password"], "cultnet.register.v0": ["schemaVersion", "nonce", "email", "password", "name"], "cultnet.verify.v0": ["schemaVersion", "nonce", "session"], @@ -355,10 +366,10 @@ def wire_message_schema_properties(schema_version: str) -> dict[str, Any]: "supportsSchemaCatalog": {"type": "boolean"}, }, "cultnet.error.v0": { - "messageId": common["messageId"], - "error": {"type": "string"}, - "code": {"type": "string"}, - "details": {"type": "object", "additionalProperties": True}, + "error": {"type": "string", "minLength": 1}, + "routingHint": {"type": ["object", "null"]}, + "code": {"type": ["string", "null"], "enum": [None, *CULTNET_ERROR_CODES]}, + "details": {"type": ["object", "null"]}, }, "cultnet.login.v0": { "nonce": {"type": "string", "minLength": 1}, diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 19c88b6a..a0378069 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -2235,9 +2235,11 @@ class CultNetTests(unittest.TestCase): self.assertEqual(delete_schema["properties"]["recordKey"]["type"], "string") error_schema = json.loads(by_version["cultnet.error.v0"]["schemaJson"]) - self.assertEqual(error_schema["required"], ["schemaVersion", "error"]) + contract = json.loads((Path(__file__).resolve().parents[3] / "contracts" / "cultnet" / "cultnet.error.schema.json").read_text(encoding="utf-8")) + self.assertEqual(error_schema["required"], contract["required"]) + self.assertEqual(sorted(error_schema["properties"]), sorted(contract["properties"])) self.assertEqual(error_schema["properties"]["error"]["type"], "string") - self.assertIn("details", error_schema["properties"]) + self.assertEqual(error_schema["properties"]["code"]["enum"], contract["properties"]["code"]["enum"]) login_schema = json.loads(by_version["cultnet.login.v0"]["schemaJson"]) self.assertEqual(login_schema["required"], ["schemaVersion", "nonce", "auth", "password"]) diff --git a/packages/cultnet-py/tests/test_cultnet_error.py b/packages/cultnet-py/tests/test_cultnet_error.py new file mode 100644 index 00000000..07f92cd9 --- /dev/null +++ b/packages/cultnet-py/tests/test_cultnet_error.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +import unittest +from pathlib import Path + +import msgpack + +from cultnet_py import CULTNET_ERROR_CODES, error_message, parse_message + +# contracts/cultnet/interop/error-vectors.json: bytes captured from MessagePack C# 3.1.7 and pinned +# by cultnet-rs/tests/error_message.rs (Rust encodes to exactly these bytes). +_REPO = Path(__file__).resolve().parents[3] +_VECTORS = json.loads((_REPO / "contracts/cultnet/interop/error-vectors.json").read_text(encoding="utf-8"))["vectors"] +_SCHEMA = json.loads((_REPO / "contracts/cultnet/cultnet.error.schema.json").read_text(encoding="utf-8")) + + +class CultNetErrorMessageTests(unittest.TestCase): + def test_closed_code_set_equals_the_schema_enum(self) -> None: + schema_codes = [c for c in _SCHEMA["properties"]["code"]["enum"] if c is not None] + self.assertEqual(sorted(CULTNET_ERROR_CODES), sorted(schema_codes)) + + def test_vectors_cover_every_code(self) -> None: + self.assertEqual({v["message"]["code"] for v in _VECTORS}, set(CULTNET_ERROR_CODES)) + + def test_decodes_csharp_and_rust_written_bytes_with_every_key(self) -> None: + for vector in _VECTORS: + message = parse_message(bytes.fromhex(vector["messagePackHex"])) + self.assertEqual(message.to_wire(), vector["message"], vector["name"]) + self.assertEqual( + list(message.to_wire()), ["schemaVersion", "error", "routingHint", "code", "details"], vector["name"] + ) + + def test_reencode_is_byte_identical(self) -> None: + for vector in _VECTORS: + expected = bytes.fromhex(vector["messagePackHex"]) + self.assertEqual(parse_message(expected).to_bytes(), expected, vector["name"]) + + def test_builder_produces_the_csharp_bytes(self) -> None: + by_name = {v["name"]: bytes.fromhex(v["messagePackHex"]) for v in _VECTORS} + stale = error_message( + "cursor_stale: cursor is stale", code="cursor_stale", details={"asOf": 1, "current": 2} + ) + self.assertEqual(stale.to_bytes(), by_name["cursor_stale"]) + invalid = error_message("cursor_invalid: cannot parse cursor", code="cursor_invalid") + self.assertEqual(invalid.to_bytes(), by_name["cursor_invalid"]) + selection = error_message( + "selection_invalid: blank key entry", + code="selection_invalid", + details={"field": "keys", "value": " "}, + ) + self.assertEqual(selection.to_bytes(), by_name["selection_invalid_field_value"]) + + def test_builder_writes_nil_keys_and_accepts_a_null_code(self) -> None: + wire = error_message("boom").to_wire() + self.assertEqual(wire, {"schemaVersion": "cultnet.error.v0", "error": "boom", "routingHint": None, "code": None, "details": None}) + + def test_unknown_code_is_refused_by_builder_and_decoder(self) -> None: + with self.assertRaises(ValueError): + error_message("x", code="unsupported_schema_version") + bad = msgpack.packb( + {"schemaVersion": "cultnet.error.v0", "error": "x", "routingHint": None, "code": "nope", "details": None}, + use_bin_type=True, + ) + with self.assertRaises(ValueError): + parse_message(bad) + + def test_dropping_code_or_details_is_refused(self) -> None: + vector = next(v for v in _VECTORS if v["name"] == "cursor_stale") + for dropped in ("code", "details", "routingHint"): + wire = dict(vector["message"]) + del wire[dropped] + with self.assertRaises(ValueError, msg=dropped): + parse_message(msgpack.packb(wire, use_bin_type=True)) + + def test_malformed_details_are_refused(self) -> None: + base = dict(next(v for v in _VECTORS if v["name"] == "cursor_stale")["message"]) + for details in ( + {"field": None, "value": None, "asOf": 1}, + {"field": None, "value": None, "asOf": -1, "current": 2}, + {"field": None, "value": None, "asOf": 1, "current": 2, "extra": 1}, + ): + with self.assertRaises(ValueError, msg=str(details)): + parse_message(msgpack.packb({**base, "details": details}, use_bin_type=True)) + with self.assertRaises(ValueError): + error_message("x", code="cursor_stale", details={"asOf": 1, "bogus": 2}) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/tests/test_first_call.py b/packages/cultnet-py/tests/test_first_call.py new file mode 100644 index 00000000..9815043c --- /dev/null +++ b/packages/cultnet-py/tests/test_first_call.py @@ -0,0 +1,72 @@ +"""The Python CultNet first call from docs/start/README.md. The docs show the region between the +`snippet` markers verbatim, and test_docs_show_the_tested_snippet fails when they differ.""" + +import unittest +from pathlib import Path + +# snippet:begin +import socket +from dataclasses import asdict, dataclass + +from cultcache_py import CultCache, define_document_type +from cultnet_py import ( + apply_raw_document_record, + document_put_raw, + encode_frame, + parse_message, + read_frame, + schema_document_map, +) + + +@dataclass +class Note: + title: str + body: str + + +note_doc = define_document_type( + "note", + encode=lambda note: asdict(note), + decode=lambda raw: Note(**raw), + name="title", +) + + +def first_call() -> Note: + sender, receiver = socket.socketpair() # an in-process pipe; use socket.create_connection for TCP + + # Send: a typed document as a cultnet.document_put_raw.v0 message in one length-prefixed frame. + put = document_put_raw( + message_id="put-1", + key="note:hello", + schema_id=note_doc.catalog_entry().schema_id, + stored_at="2026-01-01T00:00:00Z", + payload=note_doc.encode_payload(Note("Hello", "sent over CultNet")), + ) + sender.sendall(encode_frame(put.to_bytes())) + + # Receive: read the frame, parse the message, apply its record to a second cache. + message = parse_message(read_frame(receiver.makefile("rb"))) + assert message.schema_version == "cultnet.document_put_raw.v0" + inbox = CultCache.builder().register_document_type(note_doc).build() + apply_raw_document_record(inbox, schema_document_map([note_doc]), message.body["document"]) + sender.close() + receiver.close() + return inbox.get_required(note_doc, "note:hello") +# snippet:end + + +class FirstCallTests(unittest.TestCase): + def test_a_typed_document_crosses_a_pipe_into_a_second_cache(self) -> None: + self.assertEqual(first_call(), Note("Hello", "sent over CultNet")) + + def test_docs_show_the_tested_snippet(self) -> None: + source = Path(__file__).read_text(encoding="utf-8") + snippet = source.split("# snippet:begin\n", 1)[1].split("# snippet:end\n", 1)[0] + docs = (Path(__file__).parents[3] / "docs" / "start" / "README.md").read_text(encoding="utf-8") + self.assertIn("```python\n" + snippet + "```", docs) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-rs/tests/error_message.rs b/packages/cultnet-rs/tests/error_message.rs index 58732295..8b36d774 100644 --- a/packages/cultnet-rs/tests/error_message.rs +++ b/packages/cultnet-rs/tests/error_message.rs @@ -211,3 +211,22 @@ fn round_trip_is_stable() { assert_eq!(decoded, message, "{}: round trip mismatch", case.name); } } + +/// `contracts/cultnet/interop/error-vectors.json` is the file the TypeScript and Python runtimes +/// decode; it must stay byte-identical to the inline C# captures above, or those runtimes would be +/// pinned to bytes this crate no longer pins. +#[test] +fn error_vectors_file_matches_the_inline_csharp_captures() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../contracts/cultnet/interop/error-vectors.json"); + let file: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read error-vectors.json")) + .expect("parse error-vectors.json"); + let vectors = file["vectors"].as_array().expect("vectors array"); + let all = cases(); + assert_eq!(vectors.len(), all.len(), "vector count differs from the inline cases"); + for (vector, case) in vectors.iter().zip(all.iter()) { + assert_eq!(vector["name"].as_str(), Some(case.name)); + assert_eq!(vector["messagePackHex"].as_str(), Some(case.csharp_hex), "{}", case.name); + } +} diff --git a/packages/cultnet-rs/tests/first_call.rs b/packages/cultnet-rs/tests/first_call.rs new file mode 100644 index 00000000..e2ef1e6f --- /dev/null +++ b/packages/cultnet-rs/tests/first_call.rs @@ -0,0 +1,77 @@ +//! The Rust CultNet first call from docs/start/README.md. The docs show the region between the +//! `snippet` markers verbatim, and `docs_show_the_tested_snippet` fails when they differ. + +use cultcache_rs::{CultCache, DatabaseEntry}; + +// snippet:begin +use std::net::{TcpListener, TcpStream}; +use std::thread; + +use cultnet_rs::{ + CultNetDocumentBinding, CultNetDocumentPutOptions, CultNetDocumentRegistry, CultNetWireContract, + TcpFramedTransportConnection, create_tcp_framed_transport_profile, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, +}; + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "note")] +struct Note { + #[cultcache(key = 0)] + title: String, + #[cultcache(key = 1)] + body: String, +} + +fn first_call() -> anyhow::Result { + let mut documents = CultNetDocumentRegistry::new(); + documents.register(CultNetDocumentBinding::for_entry_with_schema_id::( + "gamecult.first_call_note.v1".to_string(), + "gamecult.first_call_note.v1".to_string(), + ))?; + let profile = |runtime_id| create_tcp_framed_transport_profile(runtime_id, Default::default()); + + // Receiver: a loopback listener that applies the document put it receives to its own cache. + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let receiver = thread::spawn({ + let (documents, profile) = (documents.clone(), profile("receiver")); + move || -> anyhow::Result { + let (stream, _) = listener.accept()?; + let frame = TcpFramedTransportConnection::new(stream, profile).receive()?; + let message = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?; + let mut inbox = CultCache::new(); + inbox.register_entry_type::()?; + documents.apply_raw_document_put_message::(&mut inbox, &message) + } + }); + + // Sender: build a typed put message, encode it for the wire, send one frame. + let put = documents.create_raw_document_put_message( + "put-1", + "note:hello", + &Note { title: "Hello".to_string(), body: "sent over CultNet".to_string() }, + CultNetDocumentPutOptions::default(), + )?; + let bytes = encode_cultnet_message_to_vec(&put, CultNetWireContract::CultNetSchemaV0)?; + TcpFramedTransportConnection::new(TcpStream::connect(("127.0.0.1", port))?, profile("sender")) + .send("schema", &bytes)?; + + receiver.join().expect("receiver thread") +} +// snippet:end + +#[test] +fn a_typed_document_crosses_loopback_into_a_second_cache() { + let note = first_call().unwrap(); + assert_eq!(note, Note { title: "Hello".to_string(), body: "sent over CultNet".to_string() }); +} + +#[test] +fn docs_show_the_tested_snippet() { + let source = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/first_call.rs")).unwrap(); + let begin = source.find("// snippet:begin\n").unwrap() + "// snippet:begin\n".len(); + let end = source.find("// snippet:end\n").unwrap(); + let docs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../docs/start/README.md")).unwrap(); + let block = format!("```rust\n{}```", &source[begin..end]); + assert!(docs.contains(&block), "docs/start/README.md no longer shows tests/first_call.rs verbatim"); +} diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 31188790..fc225627 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -165,9 +165,38 @@ export interface CultNetLoginSuccessMessage { session: string; } +/** The closed refusal-code set of cultnet.error.v0 (contracts/cultnet/cultnet.error.schema.json). */ +export const CULTNET_ERROR_CODES = [ + "selection_invalid", + "cursor_stale", + "cursor_invalid", + "reference_outside_target", + "variant_unsupported", + "unowned_schema", + "write_not_authorized", +] as const; + +export type CultNetErrorCode = (typeof CULTNET_ERROR_CODES)[number]; + +export interface CultNetErrorRoutingHint { + shard: Record | null; + reason: string; +} + +export interface CultNetErrorDetails { + field: string | null; + value: string | null; + asOf: number | null; + current: number | null; +} + +/** Every key is written on the wire, null included (the C# map formatter never omits a key). */ export interface CultNetErrorMessage { schemaVersion: "cultnet.error.v0"; error: string; + routingHint: CultNetErrorRoutingHint | null; + code: CultNetErrorCode | null; + details: CultNetErrorDetails | null; } export interface CultNetSampleChangeNameMessage { @@ -842,6 +871,10 @@ function parseGameCultNetworkingMessage(input: unknown): CultNetMessage { return { schemaVersion: "cultnet.error.v0", error: requireLegacyString(payload[0], "ErrorMessage.Error"), + // The legacy union ErrorMessage carries only Error; the R-N fields exist on cultnet.schema.v0 alone. + routingHint: null, + code: null, + details: null, }; case 5: return { diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index c321bd07..af53a437 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -13,7 +13,11 @@ export { type CultNetDatabaseChangeRawMessage, type CultNetRawDocumentRecord, type CultNetRawPayloadEncoding, + CULTNET_ERROR_CODES, + type CultNetErrorCode, + type CultNetErrorDetails, type CultNetErrorMessage, + type CultNetErrorRoutingHint, type CultNetHelloMessage, type CultNetLoginMessage, type CultNetLoginSuccessMessage, diff --git a/packages/cultnet-ts/src/replication.ts b/packages/cultnet-ts/src/replication.ts index a332e4ba..9676d0f0 100644 --- a/packages/cultnet-ts/src/replication.ts +++ b/packages/cultnet-ts/src/replication.ts @@ -127,10 +127,11 @@ export class CultNetDocumentRegistry { recordKey: documentKey, storedAt: options.storedAt ?? new Date().toISOString(), payload: parsed, - sourceRuntimeId: options.sourceRuntimeId, - sourceAgentId: options.sourceAgentId, - sourceRole: options.sourceRole, - tags: options.tags, + // Absent optional fields must be omitted, not set to undefined: MessagePack encodes undefined as nil and the peer rejects it. + ...(options.sourceRuntimeId !== undefined && { sourceRuntimeId: options.sourceRuntimeId }), + ...(options.sourceAgentId !== undefined && { sourceAgentId: options.sourceAgentId }), + ...(options.sourceRole !== undefined && { sourceRole: options.sourceRole }), + ...(options.tags !== undefined && { tags: options.tags }), }, }; } diff --git a/packages/cultnet-ts/test/cultnet-error.test.ts b/packages/cultnet-ts/test/cultnet-error.test.ts new file mode 100644 index 00000000..d185de8b --- /dev/null +++ b/packages/cultnet-ts/test/cultnet-error.test.ts @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { decode, encode } from "@msgpack/msgpack"; + +import { + CULTNET_ERROR_CODES, + cultNetSchemas, + encodeCultNetMessageForWire, + parseCultNetMessage, + type CultNetErrorMessage, +} from "../src"; + +// contracts/cultnet/interop/error-vectors.json: bytes captured from MessagePack C# 3.1.7 and pinned +// by cultnet-rs/tests/error_message.rs (Rust encodes to exactly these bytes). +const VECTORS = join(__dirname, "..", "..", "..", "..", "contracts", "cultnet", "interop", "error-vectors.json"); +const vectors = (JSON.parse(readFileSync(VECTORS, "utf8")) as { + vectors: { name: string; messagePackHex: string; message: unknown }[]; +}).vectors; + +const bytesOf = (hex: string): Uint8Array => Uint8Array.from(Buffer.from(hex, "hex")); +const hexOf = (bytes: Uint8Array): string => Buffer.from(bytes).toString("hex").toUpperCase(); +const wireOf = (message: unknown): Uint8Array => encode(encodeCultNetMessageForWire(message as CultNetErrorMessage)); + +test("the error vector file carries a case for every closed code", () => { + const codes = new Set(vectors.map((v) => (v.message as CultNetErrorMessage).code)); + for (const code of CULTNET_ERROR_CODES) { + assert.ok(codes.has(code), `no vector for ${code}`); + } +}); + +test("a null code with null routingHint and details is a valid error message (no captured bytes carry one)", () => { + const message = parseCultNetMessage({ + schemaVersion: "cultnet.error.v0", error: "boom", routingHint: null, code: null, details: null, + }); + assert.equal((message as CultNetErrorMessage).code, null); +}); + +test("the closed code union equals the schema's enum", () => { + const schemaCodes = ((cultNetSchemas.errorSchema.properties.code as { enum?: (string | null)[] }).enum ?? []).filter((c) => c !== null); + assert.deepEqual([...CULTNET_ERROR_CODES].sort(), [...schemaCodes].sort()); +}); + +test("C#/Rust-written error bytes decode into the typed message with every key", () => { + for (const vector of vectors) { + const parsed = parseCultNetMessage(decode(bytesOf(vector.messagePackHex))); + assert.deepEqual(parsed, vector.message, vector.name); + assert.deepEqual(Object.keys(parsed), ["schemaVersion", "error", "routingHint", "code", "details"], vector.name); + } +}); + +test("a parsed error message re-encodes to the same bytes", () => { + for (const vector of vectors) { + const parsed = parseCultNetMessage(decode(bytesOf(vector.messagePackHex))); + assert.equal(hexOf(wireOf(parsed)), vector.messagePackHex, vector.name); + } +}); + +test("dropping code or details breaks the round trip and the decode", () => { + const vector = vectors.find((v) => v.name === "cursor_stale")!; + for (const dropped of ["code", "details", "routingHint"]) { + const copy = { ...(decode(bytesOf(vector.messagePackHex)) as Record) }; + delete copy[dropped]; + assert.throws(() => parseCultNetMessage(copy), /cultnet\.error\.v0/, `missing ${dropped} must be refused`); + assert.notEqual(hexOf(encode(copy)), vector.messagePackHex); + } +}); + +test("a code outside the closed enum is refused, as is a malformed details object", () => { + const base = decode(bytesOf(vectors.find((v) => v.name === "cursor_stale")!.messagePackHex)) as Record; + assert.throws(() => parseCultNetMessage({ ...base, code: "unsupported_schema_version" }), /cultnet\.error\.v0/); + assert.throws(() => parseCultNetMessage({ ...base, code: 7 }), /cultnet\.error\.v0/); + assert.throws( + () => parseCultNetMessage({ ...base, details: { field: null, value: null, asOf: 1, current: 2, extra: 1 } }), + /cultnet\.error\.v0/, + ); + assert.throws( + () => parseCultNetMessage({ ...base, details: { field: null, value: null, asOf: -1, current: 2 } }), + /cultnet\.error\.v0/, + ); +}); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 2cc5b080..1bcfca0d 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -3059,3 +3059,22 @@ test("RUDP operation service admits a restarted client on the same address", asy await server.close(); } }); + +// MessagePack encodes undefined as nil, and a peer rejects a document whose optional source fields are nil. +test("document put message omits unset optional keys so the encoded wire has no nil", () => { + const definition = defineDocumentType({ type: "putnil.doc", schema: z.object({ body: z.string() }) }); + const binding = defineCultNetDocumentBinding({ definition }); + const registry = new CultNetDocumentRegistry([binding]); + + const bare = registry.createDocumentPutMessage(binding, "m", "k", { body: "x" }); + assert.deepEqual(Object.keys(bare.document).sort(), ["payload", "recordKey", "schemaId", "storedAt"]); + const decodedBare = decodeMsgpack(encode(bare)) as { document: Record }; + assert.ok(Object.values(decodedBare.document).every(value => value !== null), "no nil values on the wire"); + assert.ok(!("sourceRuntimeId" in decodedBare.document)); + + const stamped = registry.createDocumentPutMessage(binding, "m", "k", { body: "x" }, + { sourceRuntimeId: "r", sourceAgentId: "a", sourceRole: "role", tags: ["t"] }); + assert.deepEqual(decodeMsgpack(encode(stamped)), stamped); + assert.equal(stamped.document.sourceRuntimeId, "r"); + assert.deepEqual(stamped.document.tags, ["t"]); +}); diff --git a/packages/cultnet-ts/test/first-call.test.ts b/packages/cultnet-ts/test/first-call.test.ts new file mode 100644 index 00000000..b389ca95 --- /dev/null +++ b/packages/cultnet-ts/test/first-call.test.ts @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import test from "node:test"; + +// The TypeScript CultNet first-call snippet that docs/start/README.md shows: define a typed document, send it +// over a loopback TCP pipe as a cultnet.document_put_raw.v0 message, receive it into a second cache. The docs show +// the region between the `snippet` markers verbatim, except that the package import reads +// "cultnet-ts" where this file reads "../src"; the docs test below fails when they differ. +// snippet:begin +import { createServer, connect } from "node:net"; +import { z } from "zod"; +import { CultCache, defineDocumentType } from "@gamecult/cultcache-ts"; +import { + CultNetDocumentRegistry, + createTcpFramedCultNetPeer, + defineCultNetDocumentBinding, +} from "../src"; + +const noteDocument = defineDocumentType({ + type: "note", + schemaId: "gamecult.first_call_note.v1", + schemaName: "gamecult.first_call_note", + schemaVersion: "gamecult.first_call_note.v1", + schema: z.object({ title: z.string(), body: z.string() }), + name: "title", +}); +const documents = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: noteDocument })]); + +async function firstCall() { + const options = { runtimeId: "first-call", wireContract: "cultnet.schema.v0" } as const; + const inbox = CultCache.builder().withDocumentType(noteDocument).build(); // the receiver's cache + + // Receiver: a loopback listener whose peer applies each document put to the inbox. + const received = new Promise((resolve, reject) => { + const server = createServer((socket) => { + const peer = createTcpFramedCultNetPeer(socket, options); + peer.on("invalidMessage", reject); + peer.on("message", async (message) => { + if (message.schemaVersion !== "cultnet.document_put_raw.v0") return; + await documents.applyRawDocumentPutMessage(inbox, message); + peer.close(); + server.close(); + resolve(); + }); + }).listen(0, "127.0.0.1", () => { + // Sender: connect, build a typed put message, send it. + const { port } = server.address() as { port: number }; + const sender = createTcpFramedCultNetPeer(connect(port, "127.0.0.1"), options); + sender.send( + documents.createRawDocumentPutMessage(documents.get("note")!, "put-1", "note:hello", { + title: "Hello", + body: "sent over CultNet", + }), + ); + received.finally(() => sender.close()); + }); + }); + await received; + return inbox.get(noteDocument, "note:hello"); +} +// snippet:end + +test("send a typed document over loopback and receive it into a second cache", async () => { + const note = await firstCall(); + assert.deepEqual(note, { title: "Hello", body: "sent over CultNet" }); +}); + +test("docs show the tested snippet", async () => { + // Compiled to dist-test/test/first-call.test.js, so the package root is two levels up. + const source = await readFile(join(__dirname, "..", "..", "test", "first-call.test.ts"), "utf8"); + const snippet = source + .split("// snippet:begin\n")[1] + .split("// snippet:end\n")[0] + .replace('"../src"', '"cultnet-ts"'); + const docs = (await readFile(join(__dirname, "..", "..", "..", "..", "docs", "start", "README.md"), "utf8")).replace(/\r\n/g, "\n"); + assert.ok(docs.includes("```ts\n" + snippet + "```"), "docs/start/README.md no longer shows test/first-call.test.ts"); +}); diff --git a/packages/heimdall-consent/Cargo.lock b/packages/heimdall-consent/Cargo.lock new file mode 100644 index 00000000..758d85ea --- /dev/null +++ b/packages/heimdall-consent/Cargo.lock @@ -0,0 +1,716 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cultcache-rs" +version = "0.3.1" +dependencies = [ + "anyhow", + "chrono", + "cultcache-rs-derive", + "fs2", + "redb", + "rmp-serde", + "serde", + "serde_bytes", + "serde_json", + "serde_stacker", + "stacker", + "uuid", + "windows-sys 0.59.0", +] + +[[package]] +name = "cultcache-rs-derive" +version = "0.2.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "heimdall-consent" +version = "0.1.0" +dependencies = [ + "anyhow", + "cultcache-rs", + "serde", + "tempfile", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redb" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b317ef681e18f6091efbfab532eaf924a135c492d9d0299d0fbfae07e92a39b" +dependencies = [ + "libc", +] + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_stacker" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a" +dependencies = [ + "serde", + "serde_core", + "stacker", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/packages/heimdall-consent/Cargo.toml b/packages/heimdall-consent/Cargo.toml new file mode 100644 index 00000000..9c2b6729 --- /dev/null +++ b/packages/heimdall-consent/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "heimdall-consent" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false + +[lib] +name = "heimdall_consent" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +cultcache-rs = { path = "../cultcache-rs" } +serde = { version = "1", features = ["derive"] } + +[dev-dependencies] +tempfile = "3" diff --git a/packages/heimdall-consent/README.md b/packages/heimdall-consent/README.md new file mode 100644 index 00000000..3782e720 --- /dev/null +++ b/packages/heimdall-consent/README.md @@ -0,0 +1,36 @@ +# heimdall-consent + +Typed CultCache records for Heimdall's answer collector: `NoticeAcceptance`, `Answer` and `MemberTerms`, +plus the rules that need no store to state (`Answer::is_training_data`, `Answer::redacted`, +`validate`). Cut from `docs/answer-collector-consent.md` in GameCult/Heimdall PR #6 at doc commit +`aaeb175` (PR head); the doc's tables and Representation section are the contract, and its rulings are on the PR +(comments 682 and 759). Specification only: no collector, tally, export or tombstone hashing lives here. + +Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs` +(`cargo test`). Planned, not live: the collector, the tally, the training export. + +## Where each ruling lands (doc lines at aaeb175) + +| Ruling | Doc | Crate | +|---|---|---| +| Timestamps are Unix ms UTC, signed 64-bit | L118-119; `received_at` is collector-assigned, L95, L208-209 | `i64` on `accepted_at`, `received_at`, `terms_accepted_at` | +| Ids are non-empty opaque strings, `event_id` verbatim; every version field (`notice_version`, the `@N` of a sample id, `terms_version`) is a positive integer, no leading zeros, no sign, at most 4294967295 (Heimdall ruling 2026-11-07, one rule; doc line "Version rule" under Representation, Heimdall PR #6 commit d6a8d42, now in the doc at aaeb175) | L120-136 | `check_version` is the single rule; `check_sample_id` and `MemberTerms::validate` call it; `notice_version: u32` with `validate()` refusing 0; golden vector `newcomer` carries `terms_version` 4294967295, and case `one-version-rule` pins 4294967295 accepted, 4294967296 and `01` refused for `notice_version`, the sample id `@N` and `terms_version` | +| Key parts joined with U+001F; empty part or part containing it refused | L137-138 | `key()` | +| Tombstone is the exited `Answer` with `text_hash` (32 bytes); HMAC-SHA-256 keyed by a per-member salt held by the member-side service; the collector cannot compute it | L94, L165-185, L221-226 | `Answer.text_hash`; `redacted([u8; 32])` takes the service's output and refuses an answer with no `text` (L181-182); `validate()` requires 32 bytes and `exited` | +| A reply with no established notice version or scope is stored with both absent, writes no acceptance, is excluded from every use | L107-114 | `notice_version`/`scope` optional and absent together (`validate()`); vector case `no-scope` | +| A reply's version is the one pinned at its `origin_server_ts`; changed version needs a fresh first reply | L19-20, L49-51, L112-114 | acceptance key includes the version; `is_training_data` accepts only the acceptance for the answer's own version; vector case `version-change` | +| `MemberTerms` is the typed shape of the member record's `terms_version`/`terms_accepted_at`, not a second store | L68-76 | `MemberTerms` doc comment; the collector does not hold it | +| Training needs acceptance `scope = train` equal to the answer's `scope`, and `accepted_at <= received_at` | L149-156 | `is_training_data` | +| One store per scope plus a store for scopeless replies | L158-163 | not modelled; the collector's layout owns it | + +Rulings of comment 759 (doc L92, L98-105; no open questions remain): + +| Ruling | Doc | Crate | +|---|---|---| +| A live answer must have non-empty `text` and is rejected otherwise; empty text is valid only on an exited answer (text cleared, hash-only tombstone) | L92, L103-105 | `Answer::validate()` refuses a non-exited answer whose `text` is absent or empty; an exited answer holds no text. Vector cases `empty-live-rejected`, `empty-revoked` | +| `received_at` is assigned by the collector; a tie on it breaks by `event_id` compared bytewise, the greater wins as the later reply | L95, L98-101 | `Answer::supersedes`: `(received_at, event_id bytes)` order; the vector harness replaces a stored answer only when the incoming one supersedes it. Vector cases `tie-greater-first`, `tie-greater-last` | + +Not modelled (doc at aaeb175): + +- The `org.gamecult.notice` room state event (L24-37, L132-136): its content is `{version, scope}` with `version` a JSON number, and a string (including `"01"`) is malformed. The crate checks the version only as decimal text (`check_version`); reading, power-level checks and the malformed-means-`show` fallback belong to the collector. +- The registration-checkbox gap (L78-79): the `m.login.terms` checkbox records no version. The doc rules nothing on it; the crate takes `terms_version` as given. diff --git a/packages/heimdall-consent/src/lib.rs b/packages/heimdall-consent/src/lib.rs new file mode 100644 index 00000000..61ecb188 --- /dev/null +++ b/packages/heimdall-consent/src/lib.rs @@ -0,0 +1,225 @@ +//! Typed CultCache records for Heimdall's answer collector, cut from `docs/answer-collector-consent.md` +//! in GameCult/Heimdall PR #6 (doc commit aaeb175): the notice acceptance record, the answer record, +//! and the typed shape of the terms pair on the member record. Field names and types follow the +//! doc's tables and its Representation section; README.md cites the doc lines. + +use anyhow::{Result, bail}; +use cultcache_rs::DatabaseEntry; +use serde::{Deserialize, Serialize}; + +/// What a notice allows, each level including the ones below it. `Train` is declared in terms by the +/// room's notice; nothing here widens a scope. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Scope { + Show, + Study, + Train, +} + +/// Separates key parts (Representation: parts joined with U+001F; an empty part or one containing it +/// is refused, not escaped). +const SEP: char = '\u{1f}'; + +fn key(parts: &[&str]) -> Result { + if let Some(p) = parts.iter().find(|p| p.is_empty() || p.contains(SEP)) { + bail!("key part {p:?} is empty or contains the key separator"); + } + Ok(parts.join(&SEP.to_string())) +} + +fn nonempty(name: &str, value: &str) -> Result<()> { + if value.is_empty() { + bail!("{name} must be a non-empty string"); + } + Ok(()) +} + +/// The one version rule (Heimdall ruling 2026-11-07): a positive integer written without leading +/// zeros or a sign, at most 4294967295 (`u32::MAX`). `01` and `1` would be two keys for one version. +pub fn check_version(version: &str) -> Result { + match version.parse::() { + Ok(v) if v > 0 && !version.starts_with(['+', '0']) => Ok(v), + _ => bail!("version {version:?} must be a positive integer, no leading zeros, at most {}", u32::MAX), + } +} + +/// `@`, the version under `check_version`. The version is part of the id, so it is +/// part of every answer key and no key can name a sample without one. +pub fn check_sample_id(sample_id: &str) -> Result<(&str, u32)> { + let Some((sample, version)) = sample_id.rsplit_once('@') else { + bail!("sample id {sample_id:?} has no @version"); + }; + if sample.is_empty() { + bail!("sample id {sample_id:?} needs a non-empty sample"); + } + Ok((sample, check_version(version)?)) +} + +/// The first reply a member posts under a notice version. Written once, never updated. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "heimdall.notice-acceptance", schema = "HeimdallNoticeAcceptance")] +pub struct NoticeAcceptance { + #[cultcache(key = 0)] + pub member_id: String, + #[cultcache(key = 1)] + pub experiment_id: String, + #[cultcache(key = 2)] + pub notice_version: u32, + #[cultcache(key = 3)] + pub scope: Scope, + /// `origin_server_ts` of the first reply, unix milliseconds UTC. + #[cultcache(key = 4)] + pub accepted_at: i64, + #[cultcache(key = 5)] + pub event_id: String, +} + +impl NoticeAcceptance { + /// Representation: ids are non-empty strings, versions positive. + pub fn validate(&self) -> Result<()> { + nonempty("member_id", &self.member_id)?; + nonempty("experiment_id", &self.experiment_id)?; + nonempty("event_id", &self.event_id)?; + if self.notice_version == 0 { + bail!("notice_version must be positive"); + } + Ok(()) + } + + /// `(experiment_id, member_id, notice_version)`. + pub fn cache_key(experiment_id: &str, member_id: &str, notice_version: u32) -> Result { + key(&[experiment_id, member_id, ¬ice_version.to_string()]) + } +} + +/// One answer. `notice_version` and `scope` are absent together on a reply with no established +/// notice version or no matching acceptance: such a reply is stored, excluded from every use, and +/// writes no acceptance (Answer record, "A reply whose `notice_version` or `scope` is missing"). +/// Once exited the record is the tombstone: `text` absent, `text_hash` set. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "heimdall.answer", schema = "HeimdallAnswer")] +pub struct Answer { + #[cultcache(key = 0)] + pub sample_id: String, + #[cultcache(key = 1)] + pub experiment_id: String, + #[cultcache(key = 2)] + pub member_id: String, + #[cultcache(key = 3, default)] + pub notice_version: Option, + #[cultcache(key = 4, default)] + pub scope: Option, + /// Absent once exited. + #[cultcache(key = 5, default)] + pub text: Option, + /// One-way: set by redaction, never cleared. + #[cultcache(key = 6, default)] + pub exited: bool, + /// Assigned by the collector on receipt, unix milliseconds UTC, not copied from the reply (doc L95); + /// ties are broken by `supersedes`. + #[cultcache(key = 7)] + pub received_at: i64, + #[cultcache(key = 8)] + pub event_id: String, + /// The tombstone hash of `text`: HMAC-SHA-256, 32 raw bytes, computed by the member-side service + /// (it holds the salt). Present only once exited, and only if the collector held `text`. + #[cultcache(key = 9, default)] + pub text_hash: Option>, +} + +impl Answer { + /// `(experiment_id, sample_id, member_id)`; refuses a sample id without a positive version. + pub fn cache_key(experiment_id: &str, sample_id: &str, member_id: &str) -> Result { + check_sample_id(sample_id)?; + key(&[experiment_id, sample_id, member_id]) + } + + /// Representation and the invariants the table implies. + pub fn validate(&self) -> Result<()> { + nonempty("member_id", &self.member_id)?; + nonempty("experiment_id", &self.experiment_id)?; + nonempty("event_id", &self.event_id)?; + check_sample_id(&self.sample_id)?; + if self.notice_version == Some(0) { + bail!("notice_version must be positive"); + } + if self.notice_version.is_some() != self.scope.is_some() { + bail!("notice_version and scope are present together or absent together"); + } + if self.exited && self.text.is_some() { + bail!("an exited answer holds no text"); + } + if !self.exited && self.text.as_deref().is_none_or(str::is_empty) { + bail!("a live answer needs non-empty text; empty text is valid only on a revoked answer"); + } + if let Some(hash) = &self.text_hash { + if !self.exited || hash.len() != 32 { + bail!("text_hash is 32 bytes and only on an exited answer"); + } + } + Ok(()) + } + + /// Replacement rule for two answers at one key: the later reply is the greater `(received_at, + /// event_id)`. `received_at` is assigned by the collector; a tie breaks by `event_id` compared + /// bytewise, the greater winning (Heimdall PR #6 comment 759). Insertion order never decides. + pub fn supersedes(&self, earlier: &Answer) -> bool { + (self.received_at, self.event_id.as_bytes()) > (earlier.received_at, earlier.event_id.as_bytes()) + } + + /// Redaction is exit: text erased, `exited` set, the tombstone hash kept. The hash comes from the + /// member-side service, which holds the salt; this crate never computes it. A reply the collector + /// never collected has no text and gets no tombstone, so an answer without `text` is refused. + pub fn redacted(&self, text_hash: [u8; 32]) -> Result { + if self.text.is_none() { + bail!("no text held for event {:?}: nothing to tombstone", self.event_id); + } + Ok(Answer { text: None, exited: true, text_hash: Some(text_hash.to_vec()), ..self.clone() }) + } + + /// Training rule (doc "Training rule"): scope `train`; an acceptance for this answer's + /// `(experiment_id, member_id, notice_version)` whose own scope is `train` and equals the answer's, + /// with `accepted_at <= received_at`; not exited. A changed notice version needs its own acceptance, + /// so an acceptance for another version never covers the answer. No recorded scope, version or + /// acceptance means excluded. + pub fn is_training_data(&self, acceptance: Option<&NoticeAcceptance>) -> bool { + let (Some(Scope::Train), Some(version), Some(acc)) = (self.scope, self.notice_version, acceptance) else { + return false; + }; + !self.exited + && acc.notice_version == version + && acc.experiment_id == self.experiment_id + && acc.member_id == self.member_id + && acc.scope == Scope::Train + && Some(acc.scope) == self.scope + && acc.accepted_at <= self.received_at + } +} + +/// The typed shape of the `terms_version` / `terms_accepted_at` pair on the Heimdall member record +/// (newcomers only, recorded by the `m.login.terms` stage). It is not a second store and the +/// collector does not hold it. Members who joined before the stage, and Minds, have no such pair. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "heimdall.member-terms", schema = "HeimdallMemberTerms")] +pub struct MemberTerms { + #[cultcache(key = 0)] + pub member_id: String, + #[cultcache(key = 1)] + pub terms_version: String, + /// Unix milliseconds UTC. + #[cultcache(key = 2)] + pub terms_accepted_at: i64, +} + +impl MemberTerms { + pub fn validate(&self) -> Result<()> { + nonempty("member_id", &self.member_id)?; + check_version(&self.terms_version).map(drop) + } + + /// `member_id`. + pub fn cache_key(member_id: &str) -> Result { + key(&[member_id]) + } +} diff --git a/packages/heimdall-consent/tests/golden.rs b/packages/heimdall-consent/tests/golden.rs new file mode 100644 index 00000000..615d5cb4 --- /dev/null +++ b/packages/heimdall-consent/tests/golden.rs @@ -0,0 +1,470 @@ +use cultcache_rs::{CultCache, SingleFileMessagePackBackingStore}; +use heimdall_consent::*; + +const VECTORS: &str = include_str!("vectors/consent_golden.csv"); + +fn new_cache(dir: &tempfile::TempDir) -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + let _ = cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("consent.cc"))); + cache.pull_all_backing_stores().unwrap(); + cache +} + +fn scope(s: &str) -> Option { + match s { + "" => None, + "show" => Some(Scope::Show), + "study" => Some(Scope::Study), + "train" => Some(Scope::Train), + other => panic!("unknown scope {other}"), + } +} + +fn hex(s: &str) -> Vec { + (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect() +} + +fn put_answer(cache: &mut CultCache, a: &Answer) { + cache.put(Answer::cache_key(&a.experiment_id, &a.sample_id, &a.member_id).unwrap(), a).unwrap(); +} + +/// The collector's replace rule: an incoming answer replaces the stored one only if it is the later reply. +fn put_if_later(cache: &mut CultCache, a: &Answer) { + let k = Answer::cache_key(&a.experiment_id, &a.sample_id, &a.member_id).unwrap(); + if cache.get::(&k).unwrap().is_some_and(|old| !a.supersedes(&old)) { + return; + } + cache.put(k, a).unwrap(); +} + +fn training_events(cache: &CultCache) -> Vec { + let mut ids: Vec = cache + .get_all::() + .unwrap() + .iter() + .filter(|a| { + let acc = a.notice_version.and_then(|v| { + cache.get::(&NoticeAcceptance::cache_key(&a.experiment_id, &a.member_id, v).unwrap()).unwrap() + }); + a.is_training_data(acc.as_ref()) + }) + .map(|a| a.event_id.clone()) + .collect(); + ids.sort(); + ids +} + +#[test] +fn golden_vectors() { + let mut cases: Vec<(&str, Vec>)> = Vec::new(); + for line in VECTORS.lines().filter(|l| !l.starts_with('#') && !l.trim().is_empty()) { + let f: Vec<&str> = line.split(',').collect(); + assert_eq!(f.len(), 10, "row `{line}`"); + match cases.last_mut() { + Some((name, rows)) if *name == f[0] => rows.push(f), + _ => cases.push((f[0], vec![f])), + } + } + assert_eq!(cases.len(), 10, "fixture shrank"); + for (name, rows) in cases { + let dir = tempfile::tempdir().unwrap(); + let mut cache = new_cache(&dir); + for f in rows { + let (op, member, room, nv, sc, sample, payload, ts, event) = (f[1], f[2], f[3], f[4], f[5], f[6], f[7], f[8], f[9]); + let ctx = format!("case {name} op {op} event {event}"); + let ts: i64 = ts.parse().unwrap_or(0); + let nv: Option = nv.parse().ok(); + match op { + "reply" | "answer_raw" => { + if op == "reply" { + if let (Some(v), Some(s)) = (nv, scope(sc)) { + let k = NoticeAcceptance::cache_key(room, member, v).unwrap(); + if cache.get::(&k).unwrap().is_none() { + cache + .put(k, &NoticeAcceptance { + member_id: member.into(), + experiment_id: room.into(), + notice_version: v, + scope: s, + accepted_at: ts, + event_id: event.into(), + }) + .unwrap(); + } + } + } + let answer = Answer { + sample_id: sample.into(), + experiment_id: room.into(), + member_id: member.into(), + notice_version: nv, + scope: scope(sc), + text: Some(payload.into()), + exited: false, + received_at: ts, + event_id: event.into(), + text_hash: None, + }; + answer.validate().unwrap(); + put_if_later(&mut cache, &answer); + } + "reply_rejected" => { + let answer = Answer { + sample_id: sample.into(), + experiment_id: room.into(), + member_id: member.into(), + notice_version: nv, + scope: scope(sc), + text: Some(payload.into()), + exited: false, + received_at: ts, + event_id: event.into(), + text_hash: None, + }; + assert!(answer.validate().is_err(), "{ctx}: must be refused"); + } + "answer_revoked" => { + let answer = Answer { + sample_id: sample.into(), + experiment_id: room.into(), + member_id: member.into(), + notice_version: nv, + scope: scope(sc), + text: None, + exited: true, + received_at: ts, + event_id: event.into(), + text_hash: if payload.is_empty() { None } else { Some(hex(payload)) }, + }; + answer.validate().unwrap(); + put_if_later(&mut cache, &answer); + } + "expect_winner" => { + let a = cache.get_required::(&Answer::cache_key(room, sample, member).unwrap()).unwrap(); + assert_eq!((a.event_id.as_str(), a.text.as_deref()), (event, Some(payload)), "{ctx}"); + } + "redact" => { + let k = Answer::cache_key(room, sample, member).unwrap(); + let a = cache.get_required::(&k).unwrap(); + assert_eq!(a.event_id, event, "{ctx}"); + let hash: [u8; 32] = hex(payload).try_into().expect("tombstone hash is 32 bytes"); + put_answer(&mut cache, &a.redacted(hash).unwrap()); + } + "terms" => { + let terms = MemberTerms { member_id: member.into(), terms_version: payload.into(), terms_accepted_at: ts }; + terms.validate().unwrap(); + cache.put(MemberTerms::cache_key(member).unwrap(), &terms).unwrap(); + } + "expect_terms" => { + let t = cache.get_required::(&MemberTerms::cache_key(member).unwrap()).unwrap(); + assert_eq!((t.member_id.as_str(), t.terms_version.as_str(), t.terms_accepted_at), (member, payload, ts), "{ctx}"); + } + "expect_no_terms" => assert!(cache.get::(&MemberTerms::cache_key(member).unwrap()).unwrap().is_none(), "{ctx}"), + "expect_counts" => { + let want = format!( + "answers={};acceptances={}", + cache.get_all::().unwrap().len(), + cache.get_all::().unwrap().len() + ); + assert_eq!(want, payload, "{ctx}"); + } + "expect_train" => assert_eq!(training_events(&cache).join(";"), payload, "{ctx}"), + "expect_exited" => { + let a = cache.get_required::(&Answer::cache_key(room, sample, member).unwrap()).unwrap(); + assert!(a.exited && a.text.is_none(), "{ctx}"); + assert_eq!(a.text_hash, Some(hex(payload)), "{ctx}"); + assert_eq!((a.event_id.as_str(), a.received_at, a.sample_id.as_str()), (event, ts, sample), "{ctx}"); + } + "expect_acceptance" => { + let a = cache + .get_required::(&NoticeAcceptance::cache_key(room, member, nv.unwrap()).unwrap()) + .unwrap(); + assert_eq!((a.notice_version, Some(a.scope), a.accepted_at), (nv.unwrap(), scope(sc), ts), "{ctx}"); + assert_eq!(a.event_id, payload, "{ctx}"); + } + "version_ok" | "version_refused" => { + // member column names the field; payload is the version text as it arrives + let ok = match member { + "notice_version" => check_version(payload).is_ok_and(|v| { + NoticeAcceptance { + member_id: "m".into(), + experiment_id: room.into(), + notice_version: v, + scope: Scope::Train, + accepted_at: 1, + event_id: "e".into(), + } + .validate() + .is_ok() + }), + "sample_id" => check_sample_id(&format!("s@{payload}")).is_ok(), + "terms_version" => { + MemberTerms { member_id: "m".into(), terms_version: payload.into(), terms_accepted_at: 1 }.validate().is_ok() + } + other => panic!("unknown version field {other}"), + }; + assert_eq!(ok, op == "version_ok", "{ctx} field {member} version {payload}"); + } + other => panic!("unknown op {other}"), + } + } + } +} + +#[test] +fn records_survive_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let mut cache = new_cache(&dir); + let a = Answer { + sample_id: "s@1".into(), + experiment_id: "r".into(), + member_id: "m".into(), + notice_version: Some(2), + scope: Some(Scope::Train), + text: None, + exited: true, + received_at: 5, + event_id: "e".into(), + text_hash: Some(vec![7; 32]), + }; + a.validate().unwrap(); + put_answer(&mut cache, &a); + let reopened = new_cache(&dir); + assert_eq!(reopened.get_required::(&Answer::cache_key("r", "s@1", "m").unwrap()).unwrap(), a); +} + +#[test] +fn key_refuses_unversioned_sample_ids() { + for bad in ["s", "s@", "s@0", "s@x", "@1", "s@-1", "s@+1", "s@4294967296", "s@01", "s@00", "s@001"] { + assert!(Answer::cache_key("r", bad, "m").is_err(), "{bad}"); + } + assert!(Answer::cache_key("r", "s@4294967295", "m").is_ok()); + assert!(Answer::cache_key("r", "s@1", "m").unwrap() != Answer::cache_key("r", "s@2", "m").unwrap()); +} + +#[test] +fn one_version_rule_everywhere() { + for ok in ["1", "2", "4294967295"] { + let n: u32 = ok.parse().unwrap(); + assert_eq!(check_version(ok).unwrap(), n, "{ok}"); + assert_eq!(check_sample_id(&format!("s@{ok}")).unwrap(), ("s", n), "s@{ok}"); + assert!(MemberTerms { member_id: "m".into(), terms_version: ok.into(), terms_accepted_at: 1 }.validate().is_ok(), "terms {ok}"); + } + for bad in ["0", "4294967296", "4294967297", "8589934593", "99999999999999999999", "01", "+1", "-1", "", "2026-11"] { + assert!(check_version(bad).is_err(), "{bad}"); + assert!(check_sample_id(&format!("s@{bad}")).is_err(), "s@{bad}"); + assert!(MemberTerms { member_id: "m".into(), terms_version: bad.into(), terms_accepted_at: 1 }.validate().is_err(), "terms {bad}"); + } +} + +#[test] +fn notice_version_zero_is_refused_in_both_records() { + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 0, + scope: Scope::Train, + accepted_at: 1, + event_id: "e".into(), + }; + assert!(acc.validate().is_err()); + assert!(NoticeAcceptance { notice_version: u32::MAX, ..acc }.validate().is_ok()); +} + +#[test] +fn acceptance_after_the_answer_does_not_cover_it() { + let a = Answer { + sample_id: "s@1".into(), + experiment_id: "r".into(), + member_id: "m".into(), + notice_version: Some(1), + scope: Some(Scope::Train), + text: Some("t".into()), + exited: false, + received_at: 100, + event_id: "e".into(), + text_hash: None, + }; + let acc = |at| NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Train, + accepted_at: at, + event_id: "e".into(), + }; + assert!(a.is_training_data(Some(&acc(100)))); + assert!(!a.is_training_data(Some(&acc(101)))); + assert!(!a.is_training_data(None)); +} + +fn answer() -> Answer { + Answer { + sample_id: "s@1".into(), + experiment_id: "r".into(), + member_id: "m".into(), + notice_version: Some(1), + scope: Some(Scope::Train), + text: Some("t".into()), + exited: false, + received_at: 100, + event_id: "e".into(), + text_hash: None, + } +} + +#[test] +fn acceptance_for_another_notice_version_never_covers_the_answer() { + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Train, + accepted_at: 1, + event_id: "e0".into(), + }; + let mut a = answer(); + assert!(a.is_training_data(Some(&acc))); + a.notice_version = Some(2); + assert!(!a.is_training_data(Some(&acc)), "a changed notice version needs a fresh first reply"); +} + +#[test] +fn training_needs_acceptance_scope_equal_to_the_answers_scope() { + let mut acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Study, + accepted_at: 1, + event_id: "e0".into(), + }; + assert!(!answer().is_training_data(Some(&acc))); + acc.scope = Scope::Train; + assert!(answer().is_training_data(Some(&acc))); +} + +#[test] +fn a_reply_never_collected_gets_no_tombstone() { + let mut a = answer(); + a.text = None; + assert!(a.redacted([0; 32]).is_err()); + let t = answer().redacted([9; 32]).unwrap(); + assert!(t.exited && t.text.is_none() && t.text_hash == Some(vec![9; 32])); + assert!(t.redacted([9; 32]).is_err(), "exit is one-way and a tombstone has no text to hash again"); +} + +#[test] +fn validate_enforces_the_representation_rules() { + assert!(answer().validate().is_ok()); + let cases: Vec<(&str, Box)> = vec![ + ("empty member", Box::new(|a| a.member_id.clear())), + ("empty event", Box::new(|a| a.event_id.clear())), + ("version zero", Box::new(|a| a.notice_version = Some(0))), + ("version without scope", Box::new(|a| a.scope = None)), + ("scope without version", Box::new(|a| a.notice_version = None)), + ("live without text", Box::new(|a| a.text = None)), + ("live with empty text", Box::new(|a| a.text = Some(String::new()))), + ("exited with text", Box::new(|a| a.exited = true)), + ("hash on a live answer", Box::new(|a| a.text_hash = Some(vec![0; 32]))), + ("short hash", Box::new(|a| { + a.exited = true; + a.text = None; + a.text_hash = Some(vec![0; 31]); + })), + ]; + for (name, break_it) in cases { + let mut a = answer(); + break_it(&mut a); + assert!(a.validate().is_err(), "{name}"); + } + let mut revoked = answer(); + revoked.text = None; + revoked.exited = true; + assert!(revoked.validate().is_ok(), "empty text is valid on a revoked answer"); + let mut unscoped = answer(); + unscoped.notice_version = None; + unscoped.scope = None; + assert!(unscoped.validate().is_ok(), "a scopeless reply is stored"); + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 0, + scope: Scope::Show, + accepted_at: 1, + event_id: "e".into(), + }; + assert!(acc.validate().is_err()); + assert!(MemberTerms { member_id: "m".into(), terms_version: String::new(), terms_accepted_at: 1 }.validate().is_err()); +} + +#[test] +fn later_reply_is_greater_received_at_then_bytewise_event_id() { + let at = |t, e: &str| Answer { received_at: t, event_id: e.into(), ..answer() }; + assert!(at(2, "a").supersedes(&at(1, "z"))); + assert!(!at(1, "z").supersedes(&at(2, "a"))); + assert!(at(1, "a").supersedes(&at(1, "B")), "bytewise: 0x61 > 0x42"); + assert!(at(1, "z").supersedes(&at(1, "aa")), "bytewise, not by length"); + assert!(!at(1, "e").supersedes(&at(1, "e"))); +} + +#[test] +fn keys_join_the_parts_with_u001f_and_refuse_empty_or_separator_parts() { + assert_eq!(Answer::cache_key("r", "s@1", "m").unwrap(), "r\u{1f}s@1\u{1f}m"); + assert_eq!(NoticeAcceptance::cache_key("r", "m", 3).unwrap(), "r\u{1f}m\u{1f}3"); + assert_eq!(MemberTerms::cache_key("m").unwrap(), "m"); + for (experiment, sample, member) in [("", "s@1", "m"), ("r", "s@1", ""), ("r\u{1f}x", "s@1", "m"), ("r", "s\u{1f}x@1", "m"), ("r", "s@1", "m\u{1f}x")] { + assert!(Answer::cache_key(experiment, sample, member).is_err(), "{experiment:?} {sample:?} {member:?}"); + } + assert!(NoticeAcceptance::cache_key("", "m", 1).is_err()); + assert!(NoticeAcceptance::cache_key("r", "", 1).is_err()); + assert!(NoticeAcceptance::cache_key("r\u{1f}", "m", 1).is_err()); + assert!(NoticeAcceptance::cache_key("r", "m\u{1f}", 1).is_err()); + assert!(MemberTerms::cache_key("").is_err()); + assert!(MemberTerms::cache_key("m\u{1f}").is_err()); + // Parts that would collide if the separator were not refused or were `,` or empty. + assert_ne!(Answer::cache_key("a", "s@1", "bc").unwrap(), Answer::cache_key("ab", "s@1", "c").unwrap()); +} + +#[test] +fn an_acceptance_for_another_member_or_experiment_never_covers_the_answer() { + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: Scope::Train, + accepted_at: 1, + event_id: "e0".into(), + }; + assert!(answer().is_training_data(Some(&acc))); + let other_member = NoticeAcceptance { member_id: "m2".into(), ..acc.clone() }; + let other_experiment = NoticeAcceptance { experiment_id: "r2".into(), ..acc }; + assert!(!answer().is_training_data(Some(&other_member))); + assert!(!answer().is_training_data(Some(&other_experiment))); +} + +/// The doc states the rule once: the answer's scope is `train`, the acceptance's own scope is `train`, +/// and the two are equal. The code keeps all three conditions (any two imply the third, so a single +/// removal is an equivalent mutant); this pins the rule itself over every pair of scopes. +#[test] +fn only_a_train_answer_under_a_train_acceptance_is_training_data() { + let scopes = [Scope::Show, Scope::Study, Scope::Train]; + for answer_scope in scopes { + for acceptance_scope in scopes { + let a = Answer { scope: Some(answer_scope), ..answer() }; + let acc = NoticeAcceptance { + member_id: "m".into(), + experiment_id: "r".into(), + notice_version: 1, + scope: acceptance_scope, + accepted_at: 1, + event_id: "e0".into(), + }; + let want = answer_scope == Scope::Train && acceptance_scope == Scope::Train; + assert_eq!(a.is_training_data(Some(&acc)), want, "answer {answer_scope:?} under acceptance {acceptance_scope:?}"); + } + } +} diff --git a/packages/heimdall-consent/tests/vectors/consent_golden.csv b/packages/heimdall-consent/tests/vectors/consent_golden.csv new file mode 100644 index 00000000..4f6e4048 --- /dev/null +++ b/packages/heimdall-consent/tests/vectors/consent_golden.csv @@ -0,0 +1,78 @@ +# case,op,member,experiment,notice_version,scope,sample_id,payload,ts,event_id +# Ops run in order against a real CultCache; each case starts empty. Timestamps are unix ms. +# reply: first reply under a (room, member, notice_version) writes an acceptance, then the answer replaces any earlier one at (room, sample_id, member); payload is the text. +# answer_raw: stores the answer with no acceptance written. redact: replaces the answer whose event_id matches with its tombstone; payload is the 32-byte tombstone hash as 64 hex chars (the member-side service's output; the crate never computes it). +# A reply with no established notice version or scope is stored with both absent (doc L107-114, Answer record), writes no acceptance, and is excluded. answer_raw with a version but no acceptance models a lost or never-written acceptance: still excluded (doc L152-158). +# reply_rejected: validate must refuse the answer (nothing stored). answer_revoked: stores an exited answer with no text; payload is the tombstone hash (empty means none). expect_winner: the stored answer's event_id and text must equal event and payload. +# Replacement at one key: the later reply is the greater (received_at, event_id), event_id compared bytewise; input order never decides (doc L98-101 and Heimdall PR #6 comment 759). +# terms: member field terms_version (payload) and terms_accepted_at (ts). expect_terms: the stored pair for that member reads back as payload and ts; expect_no_terms: the member has none. expect_*: payload is what must hold. +# version_ok / version_refused: the member column names the field (notice_version, sample_id, terms_version), payload is the version text; a sample id is checked as s@. One rule for all three: positive integer, no leading zeros, at most 4294967295 (Heimdall PR #6 d6a8d42, doc at aaeb175). +# Expected values are written by hand from docs/answer-collector-consent.md (Heimdall PR #6, doc commit aaeb175), not computed by the crate. +versioned-key,reply,m1,room1,1,train,s1@1,first,100,e1 +versioned-key,reply,m1,room1,1,train,s1@2,second,200,e2 +versioned-key,reply,m1,room1,1,train,s1@1,replaced,300,e3 +versioned-key,expect_counts,,,,,,answers=2;acceptances=1,, +versioned-key,expect_train,,,,,,e2;e3,, +no-scope,reply,m2,room1,,,s1@1,unscoped,100,e4 +no-scope,reply,m2,room1,,,s1@2,unversioned,110,e5 +no-scope,answer_raw,m3,room1,1,train,s1@1,no-acceptance,120,e6 +no-scope,reply,m4,room1,1,study,s1@1,study-only,130,e7 +no-scope,reply,m5,room1,1,train,s1@1,trainable,140,e8 +no-scope,expect_counts,,,,,,answers=5;acceptances=2,, +no-scope,expect_train,,,,,,e8,, +redacted-reply,reply,m6,room1,1,train,s1@1,secret words,100,e9 +redacted-reply,reply,m7,room1,1,train,s1@1,kept,110,e10 +redacted-reply,redact,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,,e9 +redacted-reply,expect_exited,m6,room1,,,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,100,e9 +redacted-reply,expect_train,,,,,,e10,, +redacted-reply,expect_counts,,,,,,answers=2;acceptances=2,, +newcomer,terms,m8,,,,,4294967295,50, +newcomer,expect_terms,m8,,,,,4294967295,50, +newcomer,expect_no_terms,m9,,,,,,, +newcomer,reply,m8,room1,3,train,s1@1,hello,100,e11 +newcomer,reply,m9,room1,3,train,s1@1,veteran,110,e12 +newcomer,reply,m8,room1,4,study,s1@1,again,200,e13 +newcomer,expect_acceptance,m8,room1,3,train,,e11,100, +newcomer,expect_acceptance,m8,room1,4,study,,e13,200, +newcomer,expect_acceptance,m9,room1,3,train,,e12,110, +newcomer,expect_counts,,,,,,answers=2;acceptances=3,, +newcomer,expect_train,,,,,,e12,, +version-change,reply,m10,room1,1,train,s1@1,under v1,100,e14 +version-change,answer_raw,m10,room1,2,train,s1@2,v2 without a fresh first reply,200,e15 +version-change,expect_counts,,,,,,answers=2;acceptances=1,, +version-change,expect_train,,,,,,e14,, +version-change,reply,m10,room1,2,train,s1@3,first reply under v2,300,e16 +version-change,expect_counts,,,,,,answers=3;acceptances=2,, +version-change,expect_train,,,,,,e14;e16,, +empty-live-rejected,reply_rejected,m11,room1,,,s1@1,,100,e17 +empty-live-rejected,expect_counts,,,,,,answers=0;acceptances=0,, +empty-live-rejected,reply,m11,room1,,,s1@1,now filled,110,e18 +empty-live-rejected,expect_winner,m11,room1,,,s1@1,now filled,,e18 +empty-revoked,answer_revoked,m12,room1,1,train,s1@1,,100,e19 +empty-revoked,answer_revoked,m13,room1,1,train,s1@1,a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90,100,e20 +empty-revoked,expect_counts,,,,,,answers=2;acceptances=0,, +empty-revoked,expect_train,,,,,,,, +tie-greater-first,reply,m14,room1,,,s1@1,upper-B,500,B +tie-greater-first,reply,m14,room1,,,s1@1,lower-a,500,a +tie-greater-first,reply,m15,room1,,,s1@1,short-z,500,z +tie-greater-first,reply,m15,room1,,,s1@1,long-aa,500,aa +tie-greater-first,reply,m16,room1,,,s1@1,newer-ts-first,600,a +tie-greater-first,reply,m16,room1,,,s1@1,older-ts-last,500,z +tie-greater-first,expect_winner,m14,room1,,,s1@1,lower-a,,a +tie-greater-first,expect_winner,m15,room1,,,s1@1,short-z,,z +tie-greater-first,expect_winner,m16,room1,,,s1@1,newer-ts-first,,a +tie-greater-last,reply,m14,room1,,,s1@1,lower-a,500,a +tie-greater-last,reply,m14,room1,,,s1@1,upper-B,500,B +tie-greater-last,reply,m15,room1,,,s1@1,long-aa,500,aa +tie-greater-last,reply,m15,room1,,,s1@1,short-z,500,z +tie-greater-last,expect_winner,m14,room1,,,s1@1,lower-a,,a +tie-greater-last,expect_winner,m15,room1,,,s1@1,short-z,,z +one-version-rule,version_ok,notice_version,room1,,,,4294967295,,version_ok-notice_version-4294967295 +one-version-rule,version_refused,notice_version,room1,,,,4294967296,,version_refused-notice_version-4294967296 +one-version-rule,version_refused,notice_version,room1,,,,01,,version_refused-notice_version-01 +one-version-rule,version_ok,sample_id,room1,,,,4294967295,,version_ok-sample_id-4294967295 +one-version-rule,version_refused,sample_id,room1,,,,4294967296,,version_refused-sample_id-4294967296 +one-version-rule,version_refused,sample_id,room1,,,,01,,version_refused-sample_id-01 +one-version-rule,version_ok,terms_version,room1,,,,4294967295,,version_ok-terms_version-4294967295 +one-version-rule,version_refused,terms_version,room1,,,,4294967296,,version_refused-terms_version-4294967296 +one-version-rule,version_refused,terms_version,room1,,,,01,,version_refused-terms_version-01 diff --git a/src/GameCult.Mesh/docs/browser-verse-transport.md b/src/GameCult.Mesh/docs/browser-verse-transport.md index fcfde998..e825a0d4 100644 --- a/src/GameCult.Mesh/docs/browser-verse-transport.md +++ b/src/GameCult.Mesh/docs/browser-verse-transport.md @@ -12,7 +12,9 @@ over binary loopback CultNet WebSocket routes under an explicit `local-development` trust policy. It proves one canonical command effect, idempotent retry, shared observation, provider restart on a new physical route, browser rediscovery, retained lease resubscription, and a second -canonical command after reconnection. +canonical command after reconnection. Running the sample needs a sibling Eve +checkout (`--eve-root`); it is not part of `CultLib.sln` and does not build +standalone. The current substrate has a browser client boundary and local live proof, but not a complete public Verse proof: diff --git a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md index 5cb5b742..49a23f79 100644 --- a/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md +++ b/src/GameCult.Mesh/docs/getting-started/03-publish-an-eve-surface.md @@ -135,7 +135,9 @@ does **not** prove the network layer; the second program owns that evidence. The network program is [`samples/eve-browser-network`](../../../../samples/eve-browser-network/README.md). -Run it separately only while diagnosing that layer. A smoke against the +Running it needs a sibling Eve checkout (`--eve-root`); it is not part of +`CultLib.sln` and does not build standalone. Run it separately only while +diagnosing that layer. A smoke against the deployed Odin daemon remains a separate infrastructure gate. Its owner map and full acceptance contract are recorded in [`browser-verse-transport.md`](../browser-verse-transport.md). diff --git a/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md b/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md index ce18fdb3..581895a0 100644 --- a/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md +++ b/src/GameCult.Mesh/docs/getting-started/04-run-the-multiruntime-game.md @@ -58,7 +58,8 @@ source-tree fallbacks into application code. The process harness lives in [`samples/eve-browser-network`](../../../../samples/eve-browser-network/README.md). -Application code should copy the typed state, operation, and Eve composition +Running it needs a sibling Eve checkout (`--eve-root`); it does not build +standalone. Application code should copy the typed state, operation, and Eve composition shape—not its benchmark flags or fixture orchestration. For a TypeScript provider daemon, continue with the diff --git a/tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs b/tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs new file mode 100644 index 00000000..67c2f06d --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs @@ -0,0 +1,80 @@ +#nullable enable +using System; +using System.IO; +using NUnit.Framework; + +// snippet:begin +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Networking; +using MessagePack; + +[CultDocument("gamecult.first_call_note", "gamecult.first_call_note.v1")] +[MessagePackObject] +public sealed class Note +{ + [Key(0)] [CultName] public string Title = string.Empty; + [Key(1)] public string Body = string.Empty; +} + +public static class CultNetFirstCall +{ + public static async Task Run() + { + var inbox = new CultCache(); // the receiver's cache + var documents = new CultNetDocumentRegistry(); + var received = new TaskCompletionSource(); + + // Receive: a loopback listener that applies each raw document put to the inbox. + using var server = new TcpFramedCultNetSchemaServer(new TcpListener(IPAddress.Loopback, 0)); + server.OnCultNet(async (put, peer) => + { + await documents.ApplyRawDocumentPutMessageAsync(inbox, put); + received.TrySetResult(true); + }); + + // Send: a typed document as a cultnet.document_put_raw.v0 message. + using var client = new TcpFramedCultNetSchemaClient(); + client.Connect("127.0.0.1", server.LocalEndPoint.Port); + var handle = new CultRecordHandle(new CultRecordKey("note:hello")); + client.SendCultNet(documents.CreateRawDocumentPutMessage( + "put-1", handle, new Note { Title = "Hello", Body = "sent over CultNet" })); + + await received.Task.WaitAsync(TimeSpan.FromSeconds(5)); + return inbox.Get(handle.Key); + } +} +// snippet:end + +namespace GameCult.Networking.Tests +{ + // The C# CultNet first-call snippet that docs/start/README.md shows verbatim: define a typed document, send it + // over a loopback TCP pipe, receive it into a second cache. DocsShowTheTestedSnippet fails when they differ. + public class CultNetFirstCallTests + { + [Test] + public async Task SendATypedDocumentOverLoopbackAndReceiveItIntoASecondCache() + { + var note = await CultNetFirstCall.Run(); + Assert.That(note, Is.Not.Null); + Assert.That(note!.Title, Is.EqualTo("Hello")); + Assert.That(note.Body, Is.EqualTo("sent over CultNet")); + } + + [Test] + public void DocsShowTheTestedSnippet() + { + var source = File.ReadAllText(SourcePath()); + const string begin = "// snippet:begin\n"; + var start = source.IndexOf(begin, StringComparison.Ordinal) + begin.Length; + var snippet = source[start..source.IndexOf("// snippet:end\n", StringComparison.Ordinal)]; + var docs = File.ReadAllText(Path.Combine(Path.GetDirectoryName(SourcePath())!, "..", "..", "docs", "start", "README.md")); + Assert.That(docs.Replace("\r\n", "\n"), Does.Contain("```csharp\n" + snippet + "```"), + "docs/start/README.md no longer shows tests/GameCult.Networking.Tests/CultNetFirstCallTests.cs verbatim"); + } + + static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; + } +}