diff --git a/.github/workflows/cultmesh-portability.yml b/.github/workflows/cultmesh-portability.yml index 7ef2d543..959171ce 100644 --- a/.github/workflows/cultmesh-portability.yml +++ b/.github/workflows/cultmesh-portability.yml @@ -1,11 +1,18 @@ name: CultMesh portability and performance +# The forge runs one job at a time and this workflow holds the slot 14 to 30 minutes, so it runs once per change: +# on the pull request (never again on the push that merges it; a break that merges from a stale base waits for the +# schedule), skipped for design-note-only changes, and on a +# daily schedule so main is still exercised when nobody opens a pull request. on: - push: - branches: - - main - - "codex/**" pull_request: + paths-ignore: + # Not `**/*.md` or `docs/**`: docs/start/README.md is read by the first-call tests of every runtime. + - "docs/*.md" + - "research/**" + - ".voidbot/**" + schedule: + - cron: "17 3 * * *" workflow_dispatch: concurrency: @@ -115,6 +122,26 @@ jobs: - name: Verify the CultCache document owner with the TypeScript cache run: npm run test --workspace packages/cultcache-ts + # The Python cache's unit suite ran only in cultnet-interop.yml (windows-latest, dispatch-only), so on Linux + # it ran nowhere. It sits after the TypeScript cache's test so the built cultcache-ts store holder exists for the + # cross-runtime store-lock test, which skips without it. + - name: Verify the CultCache document owner with the Python cache + run: python -m unittest discover -s packages/cultcache-py/tests + + # cargo test for packages/cultcache-rs ran in no workflow: the Rust cache's own unit and doc tests were unwatched + # (only the interop example built, inside the C# store tests). The target directory is under RUNNER_TEMP, away from the checkout. + - name: Verify the CultCache document owner with the Rust cache + shell: bash + run: CARGO_TARGET_DIR="$RUNNER_TEMP/cultcache-rs-target" cargo test --manifest-path packages/cultcache-rs/Cargo.toml + + # tests/GameCult.Caching.Tests, the C# reference cache's own suite (about 1,260 tests, under a minute), ran in no + # forge workflow; only the Mesh tests above exercised the reference. + # The forge runner is globalization-invariant, so ADeterministicIdIsTheSameUnderEveryCulture... cannot construct a + # culture and throws CultureNotFoundException; it is excluded here and still runs wherever ICU is present + # (variants:follow_up:culture-test-invariant-image). + - name: Verify the CultCache document owner with the C# reference + run: dotnet test tests/GameCult.Caching.Tests/GameCult.Caching.Tests.csproj --filter "FullyQualifiedName!~ADeterministicIdIsTheSameUnderEveryCulture" --logger "console;verbosity=minimal" + # Packs the four TypeScript packages and installs them into a clean consumer, so a package that builds but does # not install or load fails here. - name: Verify the packed TypeScript packages install and load from a clean consumer diff --git a/.github/workflows/quic-native.yml b/.github/workflows/quic-native.yml index 6a4c9cdf..8f1fed18 100644 --- a/.github/workflows/quic-native.yml +++ b/.github/workflows/quic-native.yml @@ -9,13 +9,21 @@ name: CultMesh QUIC native # from the pinned debian:13 container below, never from a local or Yggdrasil # build. +# Only a change to the committed binaries, their manifest or this workflow can change what verify-committed checks, +# so a branch push or pull request runs it for those paths alone; a tag always runs it. on: push: branches: - main + paths: + - "packages/cultmesh-ts/native/**" + - ".github/workflows/quic-native.yml" tags: - "cultmesh-quic-native-v*" pull_request: + paths: + - "packages/cultmesh-ts/native/**" + - ".github/workflows/quic-native.yml" workflow_dispatch: concurrency: diff --git a/contracts/cultnet/cultnet-distributed-database.md b/contracts/cultnet/cultnet-distributed-database.md index 932ba861..8d35fbb6 100644 --- a/contracts/cultnet/cultnet-distributed-database.md +++ b/contracts/cultnet/cultnet-distributed-database.md @@ -140,6 +140,15 @@ A malformed put or delete is `message_invalid`, not a per-operation code: the pe - A shard snapshot represents a log sequence. Applying it replaces the local shard view and advances the replica cursor to that sequence. - Realtime subscriptions publish domain changes, not storage envelopes. +- A subscription's `database_change_raw` messages may carry `commitId` and `commitCount`: the changes of one cache + commit that this subscription delivers (`commitCount` of them, at least two). Both fields are optional and additive; a + peer that does not know them ignores them and applies each change alone. A replica that knows them applies the + commit's changes as one commit, so the replica judges the result (a swap of two unique-index values) and not the state + between records. A replica whose cache refuses a commit is faulted, never silent. `commitId` is opaque and valid only + within the subscription. The C# subscription server and database server tag commits; other runtimes adopt next. +- The C# writer sets every key of a `database_change_raw` message, nil where unset: a removed change carries `document` + nil, an added or updated one `recordKey` and `schemaId` nil. The schema accepts those nils; an added or updated change + still needs a document, and a removed one a `recordKey` and `schemaId`. - Client-owned input documents may be predicted locally only inside explicit `CultNetClientAuthorityScope` declarations. - Predicted input state is not authoritative shared state. The shard log still diff --git a/contracts/cultnet/cultnet.database-change-raw.schema.json b/contracts/cultnet/cultnet.database-change-raw.schema.json index 2725a5af..badada4a 100644 --- a/contracts/cultnet/cultnet.database-change-raw.schema.json +++ b/contracts/cultnet/cultnet.database-change-raw.schema.json @@ -10,15 +10,29 @@ "messageId": { "type": "string", "minLength": 1 }, "subscriptionId": { "type": "string", "minLength": 1 }, "changeKind": { "enum": ["added", "updated", "removed"] }, - "document": { "$ref": "./cultnet.raw-document-record.schema.json" }, - "recordKey": { "type": "string", "minLength": 1 }, - "schemaId": { "type": "string", "minLength": 1 } + "document": { + "oneOf": [{ "type": "null" }, { "$ref": "./cultnet.raw-document-record.schema.json" }] + }, + "recordKey": { "type": ["string", "null"], "minLength": 1 }, + "schemaId": { "type": ["string", "null"], "minLength": 1 }, + "commitId": { "type": ["string", "null"], "minLength": 1 }, + "commitCount": { "type": ["integer", "null"], "minimum": 2 } }, - "allOf": [ + "oneOf": [ { - "if": { "properties": { "changeKind": { "const": "removed" } } }, - "then": { "required": ["recordKey", "schemaId"] }, - "else": { "required": ["document"] } + "properties": { + "changeKind": { "const": "removed" }, + "recordKey": { "type": "string" }, + "schemaId": { "type": "string" } + }, + "required": ["recordKey", "schemaId"] + }, + { + "properties": { + "changeKind": { "enum": ["added", "updated"] }, + "document": { "$ref": "./cultnet.raw-document-record.schema.json" } + }, + "required": ["document"] } ] } diff --git a/contracts/cultnet/governance/fork-card-vectors.json b/contracts/cultnet/governance/fork-card-vectors.json index ea738e85..38b255dc 100644 --- a/contracts/cultnet/governance/fork-card-vectors.json +++ b/contracts/cultnet/governance/fork-card-vectors.json @@ -2692,6 +2692,16 @@ "error": "request_invalid: the request does not satisfy its document" }, "messagePackHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465AF726571756573745F696E76616C6964A56572726F72D93A726571756573745F696E76616C69643A20746865207265717565737420646F6573206E6F7420736174697366792069747320646F63756D656E74" + }, + { + "name": "fork_too_large", + "httpStatus": 422, + "message": { + "schemaVersion": "gamecult.fork.refusal.v0", + "code": "fork_too_large", + "error": "fork_too_large: fork gamecult-chat:question:q3 (order 3) alone is past the response budget" + }, + "messagePackHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465AE666F726B5F746F6F5F6C61726765A56572726F72D95A666F726B5F746F6F5F6C617267653A20666F726B2067616D6563756C742D636861743A7175657374696F6E3A713320286F7264657220332920616C6F6E6520697320706173742074686520726573706F6E736520627564676574" } ], "unavailable": [ @@ -3974,6 +3984,344 @@ "next_cursor": "c4" }, "messagePackHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A473343132AA6F70656E5F636F756E74FFAE7265636F726465645F636F756E7402A5666F726B739283AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7135A57469746C65A6466F726B2035A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657205AC6F7074696F6E735F68617368D94031366432306136646334346236636330306166643337386539663063386235316466653562303765383639653161326161393538386465316465353837666162A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7135AC6F7074696F6E735F68617368D94031366432306136646334346236636330306166643337386539663063386235316466653562303765383639653161326161393538386465316465353837666162B1636F756E7465645F616E737765725F6964A26E31A7616E737765727391DE0010AD736368656D6156657273696F6EB767616D6563756C742E6465736B5F616E737765722E7630A9616E737765725F6964A26E31A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7135A76F7574636F6D65A7436F756E746564A26F66C0A87265747261637473C0A4636F6465C0AC6F7074696F6E735F68617368D94031366432306136646334346236636330306166643337386539663063386235316466653562303765383639653161326161393538386465316465353837666162A663686F696365A26E6FA46E6F7465D9524B656570206974206C6F63616C3A206120736C6F74206D61726B2069732074686520617574686F7227732063616C6C2C20616E642073686172696E672069742069732061207365706172617465206163742EAB616E7377657265645F6174B4323032362D31312D31325431393A30323A30305AAB72656365697665645F6174B4323032362D31312D31325431393A30323A30355AAF636C61696D5F696E5F667574757265C2A8616E737765726572D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A9617574686F72697479A84F70657261746F72AA726571756573745F6964D92430303030303032302D303030302D343030302D383030302D30303030303030303030323083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7134A57469746C65A6466F726B2034A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657204AC6F7074696F6E735F68617368D94061363436303432333766393036623666656339636232393238363336383564393636616266383034386632373661666132346364346631653163346337346161A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7134AC6F7074696F6E735F68617368D94061363436303432333766393036623666656339636232393238363336383564393636616266383034386632373661666132346364346631653163346337346161B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72A26334" + }, + { + "name": "two_cards_share_one_order", + "why": "rule: order is a unique rank per Mind; the refusal names both fork ids (not expressible in the schema)", + "message": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s412", + "open_count": 4, + "recorded_count": 2, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q5", + "title": "Fork 5", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 5, + "options_hash": "16d20a6dc44b6cc00afd378e9f0c8b51dfe5b07e869e1a2aa9588de1de587fab" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q5", + "options_hash": "16d20a6dc44b6cc00afd378e9f0c8b51dfe5b07e869e1a2aa9588de1de587fab", + "counted_answer_id": "n1", + "answers": [ + { + "schemaVersion": "gamecult.desk_answer.v0", + "answer_id": "n1", + "fork_id": "gamecult-chat:question:q5", + "outcome": "Counted", + "of": null, + "retracts": null, + "code": null, + "options_hash": "16d20a6dc44b6cc00afd378e9f0c8b51dfe5b07e869e1a2aa9588de1de587fab", + "choice": "no", + "note": "Keep it local: a slot mark is the author's call, and sharing it is a separate act.", + "answered_at": "2026-11-12T19:02:00Z", + "received_at": "2026-11-12T19:02:05Z", + "claim_in_future": false, + "answerer": "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f", + "authority": "Operator", + "request_id": "00000020-0000-4000-8000-000000000020" + } + ] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q4", + "title": "Fork 4", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 4, + "options_hash": "a64604237f906b6fec9cb292863685d966abf8048f276afa24cd4f1e1c4c74aa" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q4", + "options_hash": "a64604237f906b6fec9cb292863685d966abf8048f276afa24cd4f1e1c4c74aa", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q4b", + "title": "Fork 4", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 4, + "options_hash": "a64604237f906b6fec9cb292863685d966abf8048f276afa24cd4f1e1c4c74aa" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q4b", + "options_hash": "a64604237f906b6fec9cb292863685d966abf8048f276afa24cd4f1e1c4c74aa", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "closed", + "ruling": "cultlib-gaps:resolution:question.q3.n1", + "ruled_from": [ + "n1" + ], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": "n1", + "answers": [ + { + "schemaVersion": "gamecult.desk_answer.v0", + "answer_id": "n1", + "fork_id": "gamecult-chat:question:q3", + "outcome": "Counted", + "of": null, + "retracts": null, + "code": null, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "choice": "yes", + "note": "Keep it local: a slot mark is the author's call, and sharing it is a separate act.", + "answered_at": "2026-11-12T19:02:00Z", + "received_at": "2026-11-12T19:02:05Z", + "claim_in_future": false, + "answerer": "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f", + "authority": "Operator", + "request_id": "00000021-0000-4000-8000-000000000021" + }, + { + "schemaVersion": "gamecult.desk_answer.v0", + "answer_id": "n2", + "fork_id": "gamecult-chat:question:q3", + "outcome": "Refused", + "of": null, + "retracts": null, + "code": "fork_closed", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "choice": "no", + "note": "Keep it local: a slot mark is the author's call, and sharing it is a separate act.", + "answered_at": "2026-11-12T19:02:00Z", + "received_at": "2026-11-12T19:04:05Z", + "claim_in_future": false, + "answerer": "9f8e7d6c-5b4a-4392-8170-0f1e2d3c4b5a", + "authority": "Steward", + "request_id": "00000022-0000-4000-8000-000000000022" + } + ] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "closed", + "ruling": "cultlib-gaps:resolution:question.q1.n1", + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": null + }, + "messagePackHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A473343132AA6F70656E5F636F756E7404AE7265636F726465645F636F756E7402A5666F726B739683AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7135A57469746C65A6466F726B2035A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657205AC6F7074696F6E735F68617368D94031366432306136646334346236636330306166643337386539663063386235316466653562303765383639653161326161393538386465316465353837666162A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7135AC6F7074696F6E735F68617368D94031366432306136646334346236636330306166643337386539663063386235316466653562303765383639653161326161393538386465316465353837666162B1636F756E7465645F616E737765725F6964A26E31A7616E737765727391DE0010AD736368656D6156657273696F6EB767616D6563756C742E6465736B5F616E737765722E7630A9616E737765725F6964A26E31A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7135A76F7574636F6D65A7436F756E746564A26F66C0A87265747261637473C0A4636F6465C0AC6F7074696F6E735F68617368D94031366432306136646334346236636330306166643337386539663063386235316466653562303765383639653161326161393538386465316465353837666162A663686F696365A26E6FA46E6F7465D9524B656570206974206C6F63616C3A206120736C6F74206D61726B2069732074686520617574686F7227732063616C6C2C20616E642073686172696E672069742069732061207365706172617465206163742EAB616E7377657265645F6174B4323032362D31312D31325431393A30323A30305AAB72656365697665645F6174B4323032362D31312D31325431393A30323A30355AAF636C61696D5F696E5F667574757265C2A8616E737765726572D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A9617574686F72697479A84F70657261746F72AA726571756573745F6964D92430303030303032302D303030302D343030302D383030302D30303030303030303030323083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7134A57469746C65A6466F726B2034A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657204AC6F7074696F6E735F68617368D94061363436303432333766393036623666656339636232393238363336383564393636616266383034386632373661666132346364346631653163346337346161A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7134AC6F7074696F6E735F68617368D94061363436303432333766393036623666656339636232393238363336383564393636616266383034386632373661666132346364346631653163346337346161B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964BA67616D6563756C742D636861743A7175657374696F6E3A713462A57469746C65A6466F726B2034A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657204AC6F7074696F6E735F68617368D94061363436303432333766393036623666656339636232393238363336383564393636616266383034386632373661666132346364346631653163346337346161A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964BA67616D6563756C742D636861743A7175657374696F6E3A713462AC6F7074696F6E735F68617368D94061363436303432333766393036623666656339636232393238363336383564393636616266383034386632373661666132346364346631653163346337346161B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133A57469746C65A6466F726B2033A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A6636C6F736564A672756C696E67D92663756C746C69622D676170733A7265736F6C7574696F6E3A7175657374696F6E2E71332E6E31AA72756C65645F66726F6D91A26E31A56F7264657203AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532B1636F756E7465645F616E737765725F6964A26E31A7616E737765727392DE0010AD736368656D6156657273696F6EB767616D6563756C742E6465736B5F616E737765722E7630A9616E737765725F6964A26E31A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133A76F7574636F6D65A7436F756E746564A26F66C0A87265747261637473C0A4636F6465C0AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532A663686F696365A3796573A46E6F7465D9524B656570206974206C6F63616C3A206120736C6F74206D61726B2069732074686520617574686F7227732063616C6C2C20616E642073686172696E672069742069732061207365706172617465206163742EAB616E7377657265645F6174B4323032362D31312D31325431393A30323A30305AAB72656365697665645F6174B4323032362D31312D31325431393A30323A30355AAF636C61696D5F696E5F667574757265C2A8616E737765726572D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A9617574686F72697479A84F70657261746F72AA726571756573745F6964D92430303030303032312D303030302D343030302D383030302D303030303030303030303231DE0010AD736368656D6156657273696F6EB767616D6563756C742E6465736B5F616E737765722E7630A9616E737765725F6964A26E32A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133A76F7574636F6D65A752656675736564A26F66C0A87265747261637473C0A4636F6465AB666F726B5F636C6F736564AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532A663686F696365A26E6FA46E6F7465D9524B656570206974206C6F63616C3A206120736C6F74206D61726B2069732074686520617574686F7227732063616C6C2C20616E642073686172696E672069742069732061207365706172617465206163742EAB616E7377657265645F6174B4323032362D31312D31325431393A30323A30305AAB72656365697665645F6174B4323032362D31312D31325431393A30343A30355AAF636C61696D5F696E5F667574757265C2A8616E737765726572D92439663865376436632D356234612D343339322D383137302D306631653264336334623561A9617574686F72697479A753746577617264AA726571756573745F6964D92430303030303032322D303030302D343030302D383030302D30303030303030303030323283AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132A57469746C65A6466F726B2032A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657202AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131A57469746C65A6466F726B2031A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A6636C6F736564A672756C696E67D92663756C746C69622D676170733A7265736F6C7574696F6E3A7175657374696F6E2E71312E6E31AA72756C65645F66726F6D90A56F7264657201AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72C0", + "refusal": "order 4 is shared by gamecult-chat:question:q4 and gamecult-chat:question:q4b" } ], "listScenarios": [ @@ -5796,6 +6144,406 @@ } ] }, + { + "name": "order_zero_is_legal_but_no_cursor_names_it", + "ordinal": 9, + "entries": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "steps": [ + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": null, + "limit": 3 + }, + "response": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s9", + "open_count": 4, + "recorded_count": 0, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": "c1" + }, + "responseHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A27339AA6F70656E5F636F756E7404AE7265636F726465645F636F756E7400A5666F726B739383AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133A57469746C65A6466F726B2033A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657203AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132A57469746C65A6466F726B2032A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657202AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131A57469746C65A6466F726B2031A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657201AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72A26331" + }, + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": "c1", + "limit": 3 + }, + "response": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s9", + "open_count": 4, + "recorded_count": 0, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": null + }, + "responseHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A27339AA6F70656E5F636F756E7404AE7265636F726465645F636F756E7400A5666F726B739183AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7130A57469746C65A6466F726B2030A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657200AC6F7074696F6E735F68617368D94031373737626430366666616664663665383162656435393739663337313933633462346537633364323530346639303465393133313031326630373637613964A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7130AC6F7074696F6E735F68617368D94031373737626430366666616664663665383162656435393739663337313933633462346537633364323530346639303465393133313031326630373637613964B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72C0" + } + ] + }, { "name": "a_mind_with_no_forks", "ordinal": 7, @@ -5823,6 +6571,1384 @@ ] } ], + "listShortPages": [ + { + "name": "a_short_page_resumes_below_its_last_entry", + "ordinal": 9, + "entries": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "steps": [ + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": null, + "limit": 3 + }, + "response": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s9", + "open_count": 4, + "recorded_count": 0, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": "c2" + }, + "responseHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A27339AA6F70656E5F636F756E7404AE7265636F726465645F636F756E7400A5666F726B739283AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133A57469746C65A6466F726B2033A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657203AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7133AC6F7074696F6E735F68617368D94063656438393265623636336332326533643765636536303237303830343365653534356635366239643465343432316437663562666161383663343138636532B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132A57469746C65A6466F726B2032A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657202AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72A26332" + }, + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": "c2", + "limit": 3 + }, + "response": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s9", + "open_count": 4, + "recorded_count": 0, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": null + }, + "responseHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A27339AA6F70656E5F636F756E7404AE7265636F726465645F636F756E7400A5666F726B739283AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131A57469746C65A6466F726B2031A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657201AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7130A57469746C65A6466F726B2030A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657200AC6F7074696F6E735F68617368D94031373737626430366666616664663665383162656435393739663337313933633462346537633364323530346639303465393133313031326630373637613964A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7130AC6F7074696F6E735F68617368D94031373737626430366666616664663665383162656435393739663337313933633462346537633364323530346639303465393133313031326630373637613964B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72C0" + } + ] + }, + { + "name": "a_first_entry_past_the_budget_is_fork_too_large_and_the_client_continues_below_its_order", + "ordinal": 9, + "entries": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "steps": [ + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": null, + "limit": 3 + }, + "refusal": { + "schemaVersion": "gamecult.fork.refusal.v0", + "code": "fork_too_large", + "error": "fork_too_large: fork gamecult-chat:question:q3 (order 3) alone is past the response budget" + }, + "refusalHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465AE666F726B5F746F6F5F6C61726765A56572726F72D95A666F726B5F746F6F5F6C617267653A20666F726B2067616D6563756C742D636861743A7175657374696F6E3A713320286F7264657220332920616C6F6E6520697320706173742074686520726573706F6E736520627564676574", + "httpStatus": 422, + "fork_id": "gamecult-chat:question:q3", + "order": 3 + }, + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": "c3", + "limit": 3 + }, + "response": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s9", + "open_count": 4, + "recorded_count": 0, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": null + }, + "responseHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A27339AA6F70656E5F636F756E7404AE7265636F726465645F636F756E7400A5666F726B739383AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132A57469746C65A6466F726B2032A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657202AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7132AC6F7074696F6E735F68617368D94035633763323934356538653836383963366339323862313164376530613033333634306230633636616662376663666662396537343063303139643633653739B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131A57469746C65A6466F726B2031A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657201AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361B1636F756E7465645F616E737765725F6964C0A7616E73776572739083AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7130A57469746C65A6466F726B2030A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657200AC6F7074696F6E735F68617368D94031373737626430366666616664663665383162656435393739663337313933633462346537633364323530346639303465393133313031326630373637613964A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7130AC6F7074696F6E735F68617368D94031373737626430366666616664663665383162656435393739663337313933633462346537633364323530346639303465393133313031326630373637613964B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72C0" + } + ] + }, + { + "name": "fork_too_large_on_order_zero_leaves_nothing_below", + "ordinal": 9, + "entries": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "steps": [ + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": null, + "limit": 5 + }, + "response": { + "schemaVersion": "gamecult.fork.list.v0", + "mind": "gamecult-chat", + "stamp": "s9", + "open_count": 2, + "recorded_count": 0, + "forks": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + } + ], + "next_cursor": "c1" + }, + "responseHex": "87AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E6C6973742E7630A46D696E64AD67616D6563756C742D63686174A57374616D70A27339AA6F70656E5F636F756E7402AE7265636F726465645F636F756E7400A5666F726B739183AD736368656D6156657273696F6EB667616D6563756C742E666F726B2E656E7472792E7630A463617264DE0010AD736368656D6156657273696F6EB567616D6563756C742E666F726B2E636172642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131A57469746C65A6466F726B2031A7636F6E74657874BD53686F756C64206120736C6F74206D61726B206265207368617265643FA76F7074696F6E739282A26964A3796573A474657874A953686172652069742E82A26964A26E6FA474657874AE4B656570206974206C6F63616C2EAE7265636F6D6D656E646174696F6EA26E6FA7646570656E647391D92967616D6563756C742D636861743A7175657374696F6E3A66696C652D7A6F6E652D61756469656E6365A56C65616E73C0A4666C6167C0A97261697365645F696E82A46B696E64A743757453706563A26964D92B63756C746C69622D676170733A6375745F737065633A6375742D736C6F742D746F6C6572616E63652E7234A8646563696465727391D92465306431633262332D613439352D343638372D383837392D366135623463336432653166A6737461747573A46F70656EA672756C696E67C0AA72756C65645F66726F6D90A56F7264657201AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361A67265636F726485AD736368656D6156657273696F6EB767616D6563756C742E666F726B2E7265636F72642E7630A7666F726B5F6964B967616D6563756C742D636861743A7175657374696F6E3A7131AC6F7074696F6E735F68617368D94036313935393936346264323134656131623936313535346334336563343661396661663135366464353437636335306338633136626264636435356632323361B1636F756E7465645F616E737765725F6964C0A7616E737765727390AB6E6578745F637572736F72A26331" + }, + { + "request": { + "schemaVersion": "gamecult.fork.list_request.v0", + "mind": "gamecult-chat", + "status": null, + "cursor": "c1", + "limit": 5 + }, + "refusal": { + "schemaVersion": "gamecult.fork.refusal.v0", + "code": "fork_too_large", + "error": "fork_too_large: fork gamecult-chat:question:q0 (order 0) alone is past the response budget" + }, + "refusalHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465AE666F726B5F746F6F5F6C61726765A56572726F72D95A666F726B5F746F6F5F6C617267653A20666F726B2067616D6563756C742D636861743A7175657374696F6E3A713020286F7264657220302920616C6F6E6520697320706173742074686520726573706F6E736520627564676574", + "httpStatus": 422, + "fork_id": "gamecult-chat:question:q0", + "order": 0 + } + ] + } + ], + "readTooLarge": [ + { + "name": "a_read_whose_entry_is_past_the_cap_is_fork_too_large_without_an_order", + "ordinal": 9, + "entries": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "request": { + "schemaVersion": "gamecult.fork.read_request.v0", + "mind": "gamecult-chat", + "fork_id": "gamecult-chat:question:q3" + }, + "refusal": { + "schemaVersion": "gamecult.fork.refusal.v0", + "code": "fork_too_large", + "error": "fork_too_large: fork gamecult-chat:question:q3 alone is past the response budget" + }, + "refusalHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465AE666F726B5F746F6F5F6C61726765A56572726F72D950666F726B5F746F6F5F6C617267653A20666F726B2067616D6563756C742D636861743A7175657374696F6E3A713320616C6F6E6520697320706173742074686520726573706F6E736520627564676574", + "httpStatus": 422, + "fork_id": "gamecult-chat:question:q3" + } + ], + "answerTooLarge": [ + { + "name": "a_stale_answer_whose_card_reread_is_past_the_cap_is_fork_too_large_over_a_stored_stale_refusal", + "entries": [ + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q3", + "title": "Fork 3", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 3, + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "ced892eb663c22e3d7ece602708043ee545f56b9d4e4421d7f5bfaa86c418ce2", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q2", + "title": "Fork 2", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 2, + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q2", + "options_hash": "5c7c2945e8e8689c6c928b11d7e0a033640b0c66afb7fcffb9e740c019d63e79", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q1", + "title": "Fork 1", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 1, + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q1", + "options_hash": "61959964bd214ea1b961554c43ec46a9faf156dd547cc50c8c16bbdcd55f223a", + "counted_answer_id": null, + "answers": [] + } + }, + { + "schemaVersion": "gamecult.fork.entry.v0", + "card": { + "schemaVersion": "gamecult.fork.card.v0", + "fork_id": "gamecult-chat:question:q0", + "title": "Fork 0", + "context": "Should a slot mark be shared?", + "options": [ + { + "id": "yes", + "text": "Share it." + }, + { + "id": "no", + "text": "Keep it local." + } + ], + "recommendation": "no", + "depends": [ + "gamecult-chat:question:file-zone-audience" + ], + "leans": null, + "flag": null, + "raised_in": { + "kind": "CutSpec", + "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4" + }, + "deciders": [ + "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f" + ], + "status": "open", + "ruling": null, + "ruled_from": [], + "order": 0, + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d" + }, + "record": { + "schemaVersion": "gamecult.fork.record.v0", + "fork_id": "gamecult-chat:question:q0", + "options_hash": "1777bd06ffafdf6e81bed5979f37193c4b4e7c3d2504f904e9131012f0767a9d", + "counted_answer_id": null, + "answers": [] + } + } + ], + "request": { + "schemaVersion": "gamecult.fork.answer_request.v0", + "fork_id": "gamecult-chat:question:q3", + "options_hash": "14b8ffbdb61106dbc06f69f3fc8e2072af7a205d766e4cc040e89733013d54dc", + "answer": { + "choice": "yes", + "note": "Keep it local: a slot mark is the author's call, and sharing it is a separate act." + }, + "retracted": false, + "retracts": null, + "answered_at": "2026-11-12T19:02:00Z", + "request_id": "0000a001-0000-4000-8000-00000000a001" + }, + "refusal": { + "schemaVersion": "gamecult.fork.refusal.v0", + "code": "fork_too_large", + "error": "fork_too_large: fork gamecult-chat:question:q3 alone is past the response budget" + }, + "refusalHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465AE666F726B5F746F6F5F6C61726765A56572726F72D950666F726B5F746F6F5F6C617267653A20666F726B2067616D6563756C742D636861743A7175657374696F6E3A713320616C6F6E6520697320706173742074686520726573706F6E736520627564676574", + "httpStatus": 422, + "outcome": "Refused", + "code": "stale_fork", + "counted": false, + "fork_id": "gamecult-chat:question:q3" + } + ], + "doorFaults": [ + { + "name": "read_door_answers_request_id_conflict_the_client_sees_unavailable", + "route": "read", + "doorCode": "request_id_conflict", + "httpStatus": 503, + "response": { + "schemaVersion": "gamecult.fork.unavailable.v0", + "code": "unavailable", + "error": "unavailable: the outcome is not confirmed, try again with the same request_id", + "retryAfterSeconds": 30 + }, + "responseHex": "84AD736368656D6156657273696F6EBC67616D6563756C742E666F726B2E756E617661696C61626C652E7630A4636F6465AB756E617661696C61626C65A56572726F72D94D756E617661696C61626C653A20746865206F7574636F6D65206973206E6F7420636F6E6669726D65642C2074727920616761696E2077697468207468652073616D6520726571756573745F6964B1726574727941667465725365636F6E64731E" + }, + { + "name": "list_door_answers_request_id_conflict_the_client_sees_unavailable", + "route": "list", + "doorCode": "request_id_conflict", + "httpStatus": 503, + "response": { + "schemaVersion": "gamecult.fork.unavailable.v0", + "code": "unavailable", + "error": "unavailable: the outcome is not confirmed, try again with the same request_id", + "retryAfterSeconds": 30 + }, + "responseHex": "84AD736368656D6156657273696F6EBC67616D6563756C742E666F726B2E756E617661696C61626C652E7630A4636F6465AB756E617661696C61626C65A56572726F72D94D756E617661696C61626C653A20746865206F7574636F6D65206973206E6F7420636F6E6669726D65642C2074727920616761696E2077697468207468652073616D6520726571756573745F6964B1726574727941667465725365636F6E64731E" + }, + { + "name": "answer_door_answers_request_id_conflict_the_client_sees_it", + "route": "answer", + "doorCode": "request_id_conflict", + "httpStatus": 409, + "response": { + "schemaVersion": "gamecult.fork.refusal.v0", + "code": "request_id_conflict", + "error": "request_id_conflict: this request_id was used for a different body" + }, + "responseHex": "83AD736368656D6156657273696F6EB867616D6563756C742E666F726B2E7265667573616C2E7630A4636F6465B3726571756573745F69645F636F6E666C696374A56572726F72D942726571756573745F69645F636F6E666C6963743A207468697320726571756573745F696420776173207573656420666F72206120646966666572656E7420626F6479" + } + ], "mindRoutes": { "minds": { "gamecult-chat": "huginn-gamecult-chat.sock", diff --git a/contracts/cultnet/governance/fork-card.md b/contracts/cultnet/governance/fork-card.md index c5075017..73ae333c 100644 --- a/contracts/cultnet/governance/fork-card.md +++ b/contracts/cultnet/governance/fork-card.md @@ -15,7 +15,7 @@ All documents are CultNet map-mode: every key written, nil where absent, closed | `gamecult.fork.answer_request.v0` | the answer a decider sends | the client; Bifrost forwards it unchanged to Huginn's second door | | `gamecult.desk_answer.v0` | the stored answer with one tagged outcome | Huginn only | | `gamecult.fork.answer_result.v0` | what the sender gets back for a stored answer | Bifrost, from the stored `desk_answer` | -| `gamecult.fork.refusal.v0` | a body refused at the door; nothing stored | Bifrost | +| `gamecult.fork.refusal.v0` | a body refused at the door, nothing stored (but `fork_too_large` on a stale answer, which sits over a stored `Refused(stale_fork)`) | Bifrost | | `gamecult.fork.record.v0` | a fork's answers plus the derived counted answer | a projection of the stored `desk_answer`s | | `gamecult.fork.ruling_cite.v0` | the citation a ruling carries | the Mind admitting the ruling | | `gamecult.fork.list_request.v0` | list a Mind's forks (Transport) | the desk | @@ -32,7 +32,7 @@ All documents are CultNet map-mode: every key written, nil where absent, closed - **Hashed (the shown members):** `title`, `context`, `options` (each `{id, text}`, in shown order), `recommendation` (an option id, **never nil**: the stored `question` requires `recommended`), `depends` (strings, each one of the stored question's `depends` lines verbatim, at most 8, each at most 1000 UTF-8 bytes (the stored `Line`), in shown order, empty when none; the stored kind holds prose lines, so a line may read like a reference such as `campaign:kind:label` but is hashed as the text it is). - **Not hashed:** `fork_id` (bound beside the hash), `leans` (an option id or nil), `flag`, `raised_in`, `deciders`, `status` (`open` or `closed`), `ruling` (the ruling id when a ruling closed the fork; nil when the card is open, and nil on a closed card whose question was withdrawn or otherwise resolved with no ruling), `ruled_from` (answer ids a ruling has been admitted from; non-empty only on a closed card), `order`, and `options_hash` itself. - **`deciders` is read-only data.** The list is set outside the Mind, through Bifrost's admin path, and **Bifrost sends it per call**: Huginn keeps no copy (M5). The card projection copies the list the call carried. **The list holds one to 64 ids** (Huginn's bound), on the card and in the per-call list alike (vectors `deciders_at_the_cap_of_64`, `deciders_65`); the per-call list has no schema here, so the bound is stated for the cut that builds it. A question's own `audience` may only narrow it. A Mind never writes it, so a Mind cannot add itself and answer its own fork. -- Rules a schema cannot say, each with a negative vector: a `depends` line is at most 1000 UTF-8 bytes (the schema bounds 8 lines and 1000 characters; bytes need the rule); option ids are unique; `recommendation` and `leans` name an option; deciders are unique; an open card names no `ruling` and its `ruled_from` is empty (a ruling that answers a fork closes it); `ruled_from` is non-empty only when `ruling` is set; a closed card may name no ruling (the question was withdrawn, superseded or deferred: step 6 reads `status`, not the ruling); `options_hash` equals the hash of the shown members. +- Rules a schema cannot say, each with a negative vector: a `depends` line is at most 1000 UTF-8 bytes (the schema bounds 8 lines and 1000 characters; bytes need the rule); option ids are unique; `order` is unique among a Mind's cards (a list with two cards on one `order` is refused, naming both fork ids); `recommendation` and `leans` name an option; deciders are unique; an open card names no `ruling` and its `ruled_from` is empty (a ruling that answers a fork closes it); `ruled_from` is non-empty only when `ruling` is set; a closed card may name no ruling (the question was withdrawn, superseded or deferred: step 6 reads `status`, not the ruling); `options_hash` equals the hash of the shown members. ### Options hash (canonical bytes) @@ -105,8 +105,8 @@ Source: the desk's client needs (Ratatoskr PR #24 section 12, `ratatoskr-desk-sp | call | route (proposal) | request | response | token | |---|---|---|---|---| | 1 list | `GET /api/desk/{mind}/forks?status=&cursor=&limit=` | `list_request`: `mind` from the path, the rest from the query string | `list` | audience `bifrost-desk`, scope `desk.read` | -| 2 read | `GET /api/desk/{mind}/forks/{fork_id}` (the id percent-encoded) | `read_request`, its `mind` and `fork_id` from the path | `entry`, or `refusal` `fork_unknown` (404) | `desk.read` | -| 3 answer | `POST /api/desk/{mind}/answers` | `answer_request` as the body; `fork_id` stays in the body, the Mind is the route's and is no body field | `answer_result` (200 for `Counted`, `Revision` and `Retraction`; a stored `Refused` answer keeps the `answer_result` body and takes its code's status from the counting-rules table: `stale_fork` 409, `not_a_decider` 403, `fork_closed` 409, `choice_invalid` 400, `bad_retraction` 400, `already_retracted` 409, as the vectors' `results` pin; a replay returns the same status as the first call; the desk reads `outcome` and `code` from the body), or a `refusal`, or `unavailable` | audience `bifrost-desk`, scope `desk.answer` | +| 2 read | `GET /api/desk/{mind}/forks/{fork_id}` (the id percent-encoded) | `read_request`, its `mind` and `fork_id` from the path | `entry`, or `refusal` `fork_unknown` (404) or `fork_too_large` (422) | `desk.read` | +| 3 answer | `POST /api/desk/{mind}/answers` | `answer_request` as the body; `fork_id` stays in the body, the Mind is the route's and is no body field | `answer_result` (200 for `Counted`, `Revision` and `Retraction`; a stored `Refused` answer keeps the `answer_result` body and takes its code's status from the counting-rules table: `stale_fork` 409, `not_a_decider` 403, `fork_closed` 409, `choice_invalid` 400, `bad_retraction` 400, `already_retracted` 409, as the vectors' `results` pin; a replay returns the same status as the first call; the desk reads `outcome` and `code` from the body), or a `refusal` (`fork_unknown`, or `fork_too_large` when a stale answer's card re-read is past the cap), or `unavailable` | audience `bifrost-desk`, scope `desk.answer` | **The `{mind}` segment** (Heimdall's rulings M1 to M5; vectors `transport.mindRoutes`). @@ -118,15 +118,15 @@ Source: the desk's client needs (Ratatoskr PR #24 section 12, `ratatoskr-desk-sp Calls 1 and 2 are enough for a read-only desk. Every response is the pinned MessagePack document or its JSON form with the same keys (open choice 1 of `vote-api.md`: the media type is unsettled). -**Auth, in order, per route.** (1) A bearer, else `unauthenticated` (401). (2) The token's audience is `bifrost-desk`: **the Matrix Authentication Service token chat holds today is refused, and so is any `votes.*` token** (audience `bifrost-votes`), both as `unauthenticated`. (3) The scope the route needs, else `insufficient_scope` (403): `desk.read` for calls 1 and 2, `desk.answer` for call 3, and neither implies the other. (4) The account behind `sub` is an Active member, else `forbidden` (403), as in the vote API; scope is judged first. (5) For call 3 the body is then shaped (steps 1 and 2 above), so **a non-Active member who sends a bad body, including one naming `answerer` or `authority`, gets `forbidden`, not a body refusal**; a body naming either key at any depth is `body_forbidden_field` only for an Active member. (6) Only then is Huginn asked, so `unavailable` never hides an auth or body refusal. A `votes.*` token touches none of the three. Heimdall's file leaves the audience and the claims check open; `insufficient_scope`, `forbidden`, `request_invalid` (400, a list or read request that fails its document) and `request_id_conflict` are added to the door refusals. +**Auth, in order, per route.** (1) A bearer, else `unauthenticated` (401). (2) The token's audience is `bifrost-desk`: **the Matrix Authentication Service token chat holds today is refused, and so is any `votes.*` token** (audience `bifrost-votes`), both as `unauthenticated`. (3) The scope the route needs, else `insufficient_scope` (403): `desk.read` for calls 1 and 2, `desk.answer` for call 3, and neither implies the other. (4) The account behind `sub` is an Active member, else `forbidden` (403), as in the vote API; scope is judged first. (5) For call 3 the body is then shaped (steps 1 and 2 above), so **a non-Active member who sends a bad body, including one naming `answerer` or `authority`, gets `forbidden`, not a body refusal**; a body naming either key at any depth is `body_forbidden_field` only for an Active member. (6) Only then is Huginn asked, so `unavailable` never hides an auth or body refusal. A `votes.*` token touches none of the three. Heimdall's file leaves the audience and the claims check open; `insufficient_scope`, `forbidden`, `request_invalid` (400, a list or read request that fails its document; an unknown key, `authority` included, is `request_invalid` here, whatever Huginn's own decode does with it) and `request_id_conflict` are added to the door refusals, and so is `fork_too_large` (422, list and read; see List and Read). **`request_id_conflict` is answer-only**: the idempotency key exists only on the answer. A door `request_id_conflict` on a read or list is a door fault and reaches the client as 503 `unavailable`, never as `request_id_conflict` (Persona, 2026-11-11, agreed with Heimdall; vectors `transport.doorFaults`, each carrying the client response's bytes). The `unavailable` text's "same request_id" retry hint is for answers; reads and lists carry no request_id.. -**List.** `list_request`: `{mind, status, cursor, limit}`. `mind` is the route's `{mind}`, a Mind id from the configured list. `status` is `open`, `recorded` or nil for both; `recorded` means a closed card. Cards come **newest first by `order`** (the admission ordinal). `limit` is 0 to 50; **`limit` 0 returns no entries, only the counts and the stamp**, which is what "N open" on the Mind's header needs. `cursor` is nil or `c`: the cards whose `order` is below N; a client treats it as opaque and sends back the `next_cursor` it was given. `list`: `{mind, stamp, open_count, recorded_count, forks[entry], next_cursor}`. The counts cover every card the viewer may see, whatever `status` and `cursor` asked for. `stamp` is `s`, N the receiver's last-seen Huginn admission ordinal for the Mind's desk documents: the same stamp means the same state, so a desk polls `limit` 0 and fetches bodies only when the stamp moves. `next_cursor` is nil on the last page. An `entry` is `{card, record}`: the card and its `record` (the answers with their outcomes and the derived `counted_answer_id`). +**List.** `list_request`: `{mind, status, cursor, limit}`. `mind` is the route's `{mind}`, a Mind id from the configured list. `status` is `open`, `recorded` or nil for both; `recorded` means a closed card. Cards come **newest first by `order`**. `order` is the card's unique rank per Mind: an integer from 0 that never changes once the card exists, later questions only appending (Huginn derives it from the Mind's questions sorted by admission ordinal, then key). A Mind never holds two cards of one `order`; a list or card set that does is refused, and the refusal names both fork ids. 0 is legal, but no cursor names it (`c0` fails the cursor pattern): a page ending at order 1 returns `c1`, and the next page returns the order-0 card alone with `next_cursor` nil. `limit` is 0 to 50; **`limit` 0 returns no entries, only the counts and the stamp**, which is what "N open" on the Mind's header needs. `cursor` is nil or `c`: the cards whose `order` is below N; a client treats it as opaque and sends back the `next_cursor` it was given. `list`: `{mind, stamp, open_count, recorded_count, forks[entry], next_cursor}`. The counts cover every card the viewer may see, whatever `status` and `cursor` asked for. `stamp` is `s`, N the receiver's last-seen Huginn admission ordinal for the Mind's desk documents: the same stamp means the same state, so a desk polls `limit` 0 and fetches bodies only when the stamp moves. `next_cursor` is nil on the last page. **A page may hold fewer than `limit` entries** when the next entry would push the encoded response past the receiver's response budget (Huginn: 65536 bytes); it then holds at least one, `next_cursor` is `c` so the next page resumes below it, and the counts and the stamp are those of the full list, unchanged. **`fork_too_large`** (HTTP 422) refuses a page whose **first** entry alone is past the budget: 422 because the request is well formed and authorised but names a resource the list cannot carry, and 413 is for a request body that is too large, which this is not; retrying returns the same refusal. Its `error` names the fork id and its `order`, for example `fork_too_large: fork (order ) alone is past the response budget`. **A refusal's `error` text is informational and not pinned**: the contract pins the code, the HTTP status, that the text names the fork id (and the order on a list), and the `c` continuation on a list; the vectors' wording is a sample, and the judges check those four things, not the bytes of the text. The client continues the list with cursor `c`, the cards strictly below that order, which skips this fork; it is then readable only by id (the read route returns the whole entry, and refuses with the same code only when that entry alone is past the cap, see Read); `c` is valid only when n is at least 1, so for order 0 there is nothing below and the client has reached the end. `limit` 0 returns no entries and never refuses. (Ruled by Persona, 2026-11-11.) An `entry` is `{card, record}`: the card and its `record` (the answers with their outcomes and the derived `counted_answer_id`). -**Read.** `read_request` is `{mind, fork_id}`; the response is the `entry` for one fork of that Mind. Its `card.options_hash` is **computed by the server** from the five shown members as in "Options hash" and the client recomputes it from what it rendered; a card whose hash is not the hash of its members is no card (negative vector `card_hash_wrong`). The record's `fork_id` and `options_hash` are the card's, and `counted_answer_id` is derived, not copied (negative vectors for each). Every `desk.read` holder sees every fork's card and outcome in a configured Mind; only a decider's notes depend on the caller, going to callers in the admin-set `deciders` list, which Bifrost enforces at the door (ruling `desk-read-visibility`, option (c)); `fork_unknown` is only for a fork id that does not exist under the route's Mind (an unknown Mind answers the same, M2), so no fork is ever hidden by it, and a caller without `desk.read` is refused by the auth order above (`insufficient_scope`) before any lookup. **Bifrost applies visibility at the door; Huginn returns the whole record to the door.** Every `desk.read` holder sees cards and outcomes; notes (a decider's own words) only go to callers in the admin-set `deciders` list. The read and list shapes are unchanged and carry no audience field (`eureka-substrate:question:desk-read-visibility`, Heimdall's option (c)). A stripped read serves a decider's `note` as `note: null`, never omitted (map-mode: every key is written); `desk_answer.note` is already nullable in the schema, so a served record validates as it stands. +**Read.** `read_request` is `{mind, fork_id}`; the response is the `entry` for one fork of that Mind. Its `card.options_hash` is **computed by the server** from the five shown members as in "Options hash" and the client recomputes it from what it rendered; a card whose hash is not the hash of its members is no card (negative vector `card_hash_wrong`). The record's `fork_id` and `options_hash` are the card's, and `counted_answer_id` is derived, not copied (negative vectors for each). Every `desk.read` holder sees every fork's card and outcome in a configured Mind; only a decider's notes depend on the caller, going to callers in the admin-set `deciders` list, which Bifrost enforces at the door (ruling `desk-read-visibility`, option (c)); `fork_unknown` is only for a fork id that does not exist under the route's Mind (an unknown Mind answers the same, M2), so no fork is ever hidden by it, and a caller without `desk.read` is refused by the auth order above (`insufficient_scope`) before any lookup. **Bifrost applies visibility at the door; Huginn returns the whole record to the door.** Every `desk.read` holder sees cards and outcomes; notes (a decider's own words) only go to callers in the admin-set `deciders` list. The read and list shapes are unchanged and carry no audience field (`eureka-substrate:question:desk-read-visibility`, Heimdall's option (c)). **A read is refused `fork_too_large` (422) when the whole entry is past the reply cap** (Huginn's daemon caps every reply at 65536 bytes; the cap, not the page budget, is what a read meets). Its `error` names the fork id and has no order and no cursor to carry, because there is nothing to continue with (sample: `fork_too_large: fork alone is past the response budget`) (vector `transport.readTooLarge`; the cap is in no vector). **The answer route meets the same cap:** when an answer is `stale_fork`, Bifrost re-reads the current card to attach it, and if that read is past the cap the answer is refused `fork_too_large` (422) with the read form of the text, the same document as a read. Huginn has already stored the answer as `Refused(stale_fork)`, as for any stale answer, and it is never counted; the client is told `fork_too_large` instead of receiving the current card, and a replay of the same `request_id` gets the same 422 while the fork stays oversize. It is never `unavailable` (vector `transport.answerTooLarge`). A stripped read serves a decider's `note` as `note: null`, never omitted (map-mode: every key is written); `desk_answer.note` is already nullable in the schema, so a served record validates as it stands. **Answer.** The body is the `answer_request` and nothing else a client may set: `fork_id`, `options_hash`, `answer`, `answered_at`, `request_id` (plus `retracted` and `retracts`, written `false` and nil unless the call retracts). The response is the stored outcome: `Counted`, `Revision`, `Retraction`, or `Refused` with its code, and the **`counted_answer_id`** the fork has once this call is applied (`n` or nil; derived as in "The counted answer is derived", the same value the `record` projection carries, so the desk needs no second read). A replay returns the stored outcome with the `counted_answer_id` current at the retry, not the one it had when first stored. **A `stale_fork` refusal carries the current card** in `answer_result.card`, so the desk can show the new fork without another read. Nothing is stored for a step 1 or 2 refusal, an unknown fork (or an unknown Mind, or a fork another Mind holds) or a `request_id_conflict`. `unavailable` promises nothing either way. -**Vectors (`transport` in `fork-card-vectors.json`).** `listRequest`, `readRequest` and `entry` (valid and invalid, with bytes); `listInvalid`; `listScenarios` (five scenarios over one Mind's five cards: paged open-only, recorded-only, all, `limit` 0, an empty Mind, each response with its bytes, the cursor of page two being page one's `next_cursor`); `authCases` (twenty-two cases: the MAS token, three `votes.*` tokens, a desk token with the other route's scope, a non-Active member on each route, and Huginn down on each route, with auth and membership judged first). Both suites replay the list requests through a reference judge and the auth cases through another. `mindRoutes` (a configured list of two Minds with their sockets and forks; 57 cases over the three routes: bad and percent-encoded segments, unknown Mind against unknown fork with `sameResponse` pairs of equal bytes, the POST on a Mind that does not hold the fork, scopes that are not per Mind, and auth judged before the Mind lookup) is replayed by both suites through a reference judge that reports the refusal, the socket it would open and the Mind that stored an answer. Top-level `unavailable` pins the 503 document. Scenarios pin: an `unavailable` where Huginn never had the request, retried into a `Counted` answer; an `unavailable` where Huginn had stored it, whose retry returns the stored outcome and adds no Revision; a replayed `Refused` with its card; `request_id_conflict`; the same id from another account; and a non-Active member with a bad body, with the door down, getting `forbidden` first. A bad body is refused before the door is asked. +**Vectors (`transport` in `fork-card-vectors.json`).** `listRequest`, `readRequest` and `entry` (valid and invalid, with bytes); `listInvalid` (including `two_cards_share_one_order`, whose `refusal` is the exact text both judges must produce: `order 4 is shared by and `); `listScenarios` (six scenarios: over one Mind's five cards, paged open-only, recorded-only, all and `limit` 0; an empty Mind; and `order_zero_is_legal_but_no_cursor_names_it`, four cards of orders 3 to 0 paged by 3, whose first page ends at order 1 with `c1` and whose second page is the order-0 card with `next_cursor` nil; each response with its bytes, the cursor of page two being page one's `next_cursor`); `listShortPages` (three scenarios: a page that stops short and resumes below its last entry; `fork_too_large` on a first entry of order 3, continued with `c3`; and a short page ending at order 1 followed by `fork_too_large` on order 0, where the client is at the end; each step carries its bytes, and the receiver's budget is in no vector, so the judges check a short page as a nonempty strict prefix of the unbudgeted page and a refusal as naming the entry the page would open with); `doorFaults` (three cases: a door `request_id_conflict` is `unavailable` 503 on read and list and itself 409 on the answer); `readTooLarge` (one case: a read whose entry is past the cap, refused `fork_too_large` 422 with the read-form text and its bytes); `answerTooLarge` (one case: a stale answer whose card re-read is past the cap, the same 422 document over a stored `Refused(stale_fork)`: `outcome` `Refused`, `code` `stale_fork`, `counted` false); `authCases` (twenty-two cases: the MAS token, three `votes.*` tokens, a desk token with the other route's scope, a non-Active member on each route, and Huginn down on each route, with auth and membership judged first). Both suites replay the list requests through a reference judge and the auth cases through another. `mindRoutes` (a configured list of two Minds with their sockets and forks; 57 cases over the three routes: bad and percent-encoded segments, unknown Mind against unknown fork with `sameResponse` pairs of equal bytes, the POST on a Mind that does not hold the fork, scopes that are not per Mind, and auth judged before the Mind lookup) is replayed by both suites through a reference judge that reports the refusal, the socket it would open and the Mind that stored an answer. Top-level `unavailable` pins the 503 document. Scenarios pin: an `unavailable` where Huginn never had the request, retried into a `Counted` answer; an `unavailable` where Huginn had stored it, whose retry returns the stored outcome and adds no Revision; a replayed `Refused` with its card; `request_id_conflict`; the same id from another account; and a non-Active member with a bad body, with the door down, getting `forbidden` first. A bad body is refused before the door is asked. **Bifrost must still build:** all three routes; the `bifrost-desk` audience and the `desk.read` and `desk.answer` scopes on the OIDC token it issues to chat; the configured Mind list and its sockets (M1); the projection of cards and records from Huginn, the stamp, the cursor and the counts; the `DeskAnswer` call on the second door and the pass-through of its outcome; the decider list, sent with every answer, read and list request from the admin-set list only (M5); the `unavailable` answer. **Huginn must still build** the second door's wire, the per-call decider list it receives with each answer, read and list request, keeping no copy (M5), and the read side the projection uses (Eureka's cut spec, after her record PR). **The desk may build today** against the vectors: nothing in them needs a live service. @@ -147,7 +147,7 @@ The old desk (`index.html`) read `questions/` and wrote `answers/` in a | `lean` ("Self leans" mark) | none | `leans` (an id or nil), not hashed | | `flag` (the provisional line) | none | `flag` (a string or nil), not hashed | | `status` open or closed, `ruling` | derived from the resolution (status is never written): any resolution of the question closes it; the ruling is the one that answered it | `status`, `ruling`, both a projection (`ruling` is nil on an open card and on a card closed with no ruling) | -| `order` | none (admission ordinal) | `order` | +| `order` | none (unique rank per Mind, from 0) | `order` | | none | `raised_in`, `asked_on` | `raised_in`; `asked_on` is not carried | | none | none | `deciders` (outside the Mind), `ruled_from`, `options_hash` | | answer `{choice: key or null, note, question, updatedAt}`, one mutable document per question | none | `answer_request` then an append-only `desk_answer` | @@ -188,8 +188,10 @@ Option B, **Bifrost receives the answer and Huginn stores it** (the design in th 4. Whether `fork_unknown` and shape refusals should leave an audit record somewhere other than a `desk_answer`. 5. Whether `leans` and `flag`, which the booth shows, should join the hash. The binding file's five members do not include them; a changed lean would not make an answer stale. 6. What the person sees of a retracted counted answer when the next answer counts: this contract only says the count moves. -7. `received_at` is not in the binding file's stored fields; it is kept for `claim_in_future` and could be dropped if the ordinal alone decides order. +7. `received_at` is not in the binding file's stored fields; it is kept for `claim_in_future` and could be dropped if the `n` answer ordinal alone decided answer order. 8. The stamped `authority` field is an attribution, not an authority (above); renaming it `attribution` would say so, at the price of a new key the door must also refuse in a body. 9. The claims Bifrost verifies are PR #24's open item 5. (The audience `bifrost-desk`, the three routes and the scope names are ruled by Heimdall.) 10. The second door's wire between Bifrost and Huginn (pending Eureka's cut spec). `unavailable` is a document of its own, outside the refusal set, so a desk can retry it; calls 1 and 2 use it too when Huginn is down (vectors `huginn_down_*`). 11. While a known Mind's door is down its routes answer `unavailable` (503) and an unknown Mind answers `fork_unknown` (404), so the two are distinguishable for that period. M2 makes them identical only when every door is up; making them identical when one is down means answering `unavailable` for any `{mind}` not on the list, which tells a client nothing about which Minds exist but also hides a typo. Heimdall's to rule; the vectors do not pin it. +12. A fork's answers are unbounded in number, so a fork's entry can grow past the reply cap and become unreadable (`fork_too_large` on read, and on list when it is the first entry of a page). Whether the contract should cap the answers per fork is for Persona and Heimdall to rule; nothing here decides it. +13. The read form of `fork_too_large` is a target, not current behaviour: Huginn's door answers an over-cap read as `Refused(ResponseTooLarge{bytes, limit})`, which carries no fork id and is not `fork_too_large`. Bifrost maps it to `fork_too_large` (422) using the route's `fork_id`; until it does, a read past the cap reaches the client as 503 `unavailable`. The same holds for the answer route: a `stale_fork` answer whose card re-read is past the cap is a Bifrost target too (`fork_too_large` 422 over a stored `Refused(stale_fork)`), and until it is built the client sees 503 `unavailable`. diff --git a/contracts/cultnet/governance/fork-card.schema.json b/contracts/cultnet/governance/fork-card.schema.json index 62acc30a..bd4c21d8 100644 --- a/contracts/cultnet/governance/fork-card.schema.json +++ b/contracts/cultnet/governance/fork-card.schema.json @@ -714,6 +714,7 @@ "body_unknown_field", "body_invalid", "fork_unknown", + "fork_too_large", "insufficient_scope", "forbidden", "request_invalid", diff --git a/docs/document-variants-id-sweep.md b/docs/document-variants-id-sweep.md index 03fad9cb..74ecd023 100644 --- a/docs/document-variants-id-sweep.md +++ b/docs/document-variants-id-sweep.md @@ -4,6 +4,8 @@ Data for the per-consumer sweep in docs/document-variants-cut.md, C2. Every obje Method: a static scan of `F:\Projects\` C# files for types marked `[CultDocument(`, walked through keyed members, base classes, dictionary values and `[Union]` subtypes. Read-only; no consumer was edited. Worktree copies (`Aetheria-catalog`, `CultLib-*`, and so on) are excluded. Struct elements are values and need no id. Non-C# consumers carry the id as an ordinary member and are not listed. The scan is text-based: a document registered without a literal `[CultDocument(` attribute, or a type whose members lack an integer `[Key]`, is missed, so the sweep pass should confirm each repo by registering its types once. +A type that already has a readable string id keeps it and adds a trailing keyed string member marked `[CultElementId(nameof(ExistingId))]`; see the Element ids section of `src/GameCult.Caching/README.md`. + Lines: `Type file:line (via Owner.Member)`, paths relative to the consumer root. diff --git a/docs/nuget-packaging.md b/docs/nuget-packaging.md index 301e54eb..7894fe5e 100644 --- a/docs/nuget-packaging.md +++ b/docs/nuget-packaging.md @@ -25,5 +25,13 @@ closure through UPM rather than NuGet restore. Install the Unity package from the immutable release tag: ```text -https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v2.0.0 +https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v3.0.0 ``` + +## Rolling back across the store format + +A consumer that upgrades and writes a store holding an element id leaves a +`cultcache.store.v3` single-file store (`v5` directory store). Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. There is +no general carry-back; restore from a copy taken before upgrading, and upgrade +readers before writers. Detail: the Store format and rollback section of +`src/GameCult.Caching/README.md`. diff --git a/docs/runtime-parity-scope.md b/docs/runtime-parity-scope.md index e001bb1d..2f54e2fe 100644 --- a/docs/runtime-parity-scope.md +++ b/docs/runtime-parity-scope.md @@ -16,7 +16,7 @@ Cross-runtime parity has three layers: Every store reader refuses, by name, a header or record slot it does not understand (vectors: `tests/vectors/document-variants-c0`). Element ids: a store that can hold them declares `cultcache.store.v3`; TypeScript, - Python and Rust read it and keep the header, and readers older than ids refuse it + Python and Rust read it and keep the header, and the C# runtime decides the mark from content, and these runtimes never drop it. A C# reader from `cultlib-unity-v2.0.0` on refuses a v3 single-file store it cannot read (earlier pins do not) (vectors: `tests/vectors/document-variants-c2a`). Document variants: C# resolves them (`cultcache.store.v2`); every other runtime, CultMesh single-file reads, the directory store and CultNet refuse a variant by name diff --git a/docs/start/README.md b/docs/start/README.md index e033c935..6a7d6d74 100644 --- a/docs/start/README.md +++ b/docs/start/README.md @@ -44,7 +44,7 @@ using MessagePack; public sealed class Item { [Key(0)] [CultName] public string Name = string.Empty; - [Key(1)] [CultIndex] public string Category = string.Empty; + [Key(1)] public string Category = string.Empty; [Key(2)] public int Value; } @@ -94,7 +94,6 @@ const itemDocument = defineDocumentType({ schemaVersion: "gamecult.first_call_item.v1", schema: z.object({ name: z.string(), category: z.string(), value: z.number().int() }), name: "name", - indexes: { category: "category" }, }); const registry = defineDocumentRegistry(itemDocument); @@ -168,8 +167,8 @@ fn first_call() -> anyhow::Result<()> { ## Python first call -A document is a definition with an encoder and decoder; `name=` and `indexes=` register the name lookup -and indexes. The source is +A document is a definition with an encoder and decoder; `name=` registers the name lookup, `indexes=` unique lookups and +`roles=` many-row selection aliases. The source is [`packages/cultcache-py/tests/test_first_call.py`](../../packages/cultcache-py/tests/test_first_call.py), run by `python -m unittest tests.test_first_call` in `packages/cultcache-py` with `src` on `PYTHONPATH` and `msgpack` installed (the store needs it). A second test fails if the block below stops matching the @@ -195,7 +194,6 @@ item_doc = define_document_type( encode=lambda item: asdict(item), decode=lambda raw: Item(**raw), name="name", - indexes={"category": "category"}, ) diff --git a/docs/studio-grouping-cut.md b/docs/studio-grouping-cut.md index 770d0698..7bc8418c 100644 --- a/docs/studio-grouping-cut.md +++ b/docs/studio-grouping-cut.md @@ -33,9 +33,9 @@ Recovered from the legacy "Database Tools" window (Aetheria `d3db1730`, **The attribute is not `[CultIndex]`.** `[CultIndex]` means unique (operator, 2026-09-30; `CultDocumentContracts.cs:34`). Grouping is `CultInspectorGroupByAttribute`, in `CultInspectorAttributes.cs`, read only by `CultInspectorModel`. It changes no schema id, -catalog entry, index, or store byte. The index cut's deletion of grouping-only `[CultIndex]` -declarations (CultMesh, Geometry) adds no `[CultInspectorGroupBy]` in their place: those are -service stores, not Studio-authored catalogs. +catalog entry, index, or store byte. The index cut converts the grouping and selection `[CultIndex]` +declarations (CultMesh, Geometry) to `[CultRole]` rather than deleting them, and adds no +`[CultInspectorGroupBy]` in their place: those are service stores, not Studio-authored catalogs. ### Invariants diff --git a/packages/cultcache-py/CHANGELOG.md b/packages/cultcache-py/CHANGELOG.md index b460ab05..d440ddca 100644 --- a/packages/cultcache-py/CHANGELOG.md +++ b/packages/cultcache-py/CHANGELOG.md @@ -9,7 +9,9 @@ ### Breaking -- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- `indexes` are unique, as C# `[CultIndex]`: a write, load or `register_index` that leaves two records of a type holding one non-blank value under one alias is refused (`CultCacheError`) naming both keys and the alias, and writes nothing. Index text is the C# canonical text (`True`/`False`, integers as digits, floats as the exact expansion, blank values never indexed), so an empty or whitespace string is no longer a value. A new `roles` map names the many-row selection aliases (never indexed for lookup, same catalog `index_alias`). Classify every `indexes` key per the README before updating. The verdict is the contract, the text is informational. + +- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `push_all` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. - Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload. diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index 41c3095c..236009ed 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -73,7 +73,7 @@ store. - `register_document_type(...)` - `register_registry(...)` - `register_name_lookup(...)` -- `register_index(...)` +- `register_index(...)` (a unique index) - `add_backing_store(...)` - `add_generic_store(...)` - `pull_all_backing_stores()` @@ -95,6 +95,33 @@ store. - `delete_global(...)` - `snapshot()` +## Indexes And Roles + +`indexes=` are unique lookups (variants:ruling:unique-indexes; C# `[CultIndex]`): within one document type and one alias +no two records hold one non-blank value, judged on the result of every write (a batch is judged whole, so a swap inside +one `put_envelopes` lands) and of every load and `register_index`. A refusal is a `CultCacheError`, +`Records and () both hold unique index ''; an index value names one record.`, keys in +ordinal order, never the value; it writes nothing, and a refused load or registration installs nothing. The contract is +the verdict; the text is informational. Blank values (absent, empty, or only C# `White_Space` characters) are never +indexed and never conflict. Two types may share an alias and a value. Values compare as the C# canonical text: strings as +is (never trimmed), `True`/`False`, integers as digits, a non-integral float as its exact decimal expansion, NaN and +infinities no value. + +`roles=` name selection aliases (C# `[CultRole]`): many records may share a value, nothing is indexed or refused, +`get_by_index` on a role answers `None`, and the member's catalog `index_alias` is the same either way, so catalog bytes +do not move. An alias is declared in one of the two. `CultMeshDatabase.watch_by_index` reads `indexes` only. + +Migrating an existing `indexes` key, per key, before bumping: + +1. Nothing looks records up by the key (`get_by_index`, `get_key_by_index`, `node.get_by_index`, `watch_by_index`): move it to `roles`. +2. Something looks it up: it stays unique only when two live records can never hold one non-blank value (the record key + is derived from that value alone, or it is an outside identity: email, sku, external id). Enums, statuses, kinds, + categories, parent or foreign ids, shared hashes and timestamps are roles; select them with `get_all` plus a filter. +3. Open each real `.cc` once with the new definitions. A refusal names both keys and the alias and writes nothing. + +Misclassifying: a role declared unique makes a store that holds a shared value refuse to open until the key is +reclassified; a unique key declared a role answers no lookups and stops refusing duplicates. + ## Persistence Model Backing stores persist envelopes, not caller domain objects. The cache decodes payloads through registered document definitions and rejects unknown type discriminators. This keeps Python's dynamic runtime from turning persistence into an open polymorphic sewer with a cheerful docstring. @@ -112,6 +139,10 @@ used by the TypeScript, Rust, and C# runtimes: Large corpora should use a sharded store or a real database. A single snapshot file is a scalpel, not a forklift. +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-py from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) + ## DatabaseEntry Slot Contract For cross-runtime cache entries, use `define_database_entry_type(...)` instead diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index 8f4ac759..9c402968 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -1,9 +1,11 @@ from __future__ import annotations import functools +import math import threading +from decimal import Decimal from dataclasses import dataclass, field, replace -from typing import Any, Callable, Generic, TypeVar +from typing import Any, Callable, Generic, Iterable, TypeVar from .backing_store import BackingStore, CultCacheEnvelope from .documents import DocumentDefinition, extract_value @@ -52,6 +54,52 @@ class CultCacheError(RuntimeError): pass +# Blank is no value: empty or only C# White_Space characters (the 25 chars string.IsNullOrWhiteSpace is true for). +# An explicit set, not str.isspace(): Python also counts U+001C-001F, which C# does not, and omits nothing C# counts. +_BLANK_INDEX_CHARS = frozenset( + chr(code) + for code in ( + *range(0x09, 0x0E), + 0x20, + 0x85, + 0xA0, + 0x1680, + *range(0x2000, 0x200B), + 0x2028, + 0x2029, + 0x202F, + 0x205F, + 0x3000, + ) +) + + +def _utf16_order(text: str) -> bytes: + """Sorts as C# StringComparer.Ordinal does: by UTF-16 code unit, not code point.""" + return text.encode("utf-16-be", "surrogatepass") + + +def index_text(value: Any) -> str | None: + """The canonical text of an index value, as C# IndexValue and RenderCanonicalNumber, or None when it names no + value: a string is as is (never trimmed), a boolean is `True`/`False`, an integer is its digits, a non-integral + finite float is its exact decimal expansion, NaN and infinities and blank strings are no value.""" + if value is None: + return None + if isinstance(value, bool): # before int: bool is an int subclass + return "True" if value else "False" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if not math.isfinite(value): + return None + if value.is_integer(): + return str(int(value)) # -0.0 is 0 + text = format(Decimal(value), "f") # Decimal(float) is exact + return text.rstrip("0").rstrip(".") if "." in text else text + text = value if isinstance(value, str) else str(value) + return None if all(char in _BLANK_INDEX_CHARS for char in text) else text + + @dataclass class _State: documents: dict[str, DocumentDefinition[Any]] = field(default_factory=dict) @@ -186,6 +234,8 @@ class CultCache: @_mutating def register_index(self, document: DocumentDefinition[Any], index: str, extractor: str | Any) -> None: self._assert_registered(document) + if index in document.roles: + raise CultCacheError(f"Index '{index}' of {document.type} is declared as a role; a role is never a unique index.") index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()} index_extractors.setdefault(document.type, {})[index] = extractor self._install_lookups(self._state.name_extractors, index_extractors) @@ -195,7 +245,11 @@ class CultCache: ) -> None: """Derives every lookup under the candidate extractors, then installs extractors and lookups together. An extractor that raises on a held value installs nothing.""" - names, indexes, lookup_keys = self._derive_lookups(self._state.values, name_extractors, index_extractors) + names, indexes, lookup_keys = self._derive_lookups( + [(type, key, value) for type, values in self._state.values.items() for key, value in values.items()], + name_extractors, + index_extractors, + ) ( self._state.name_extractors, self._state.index_extractors, @@ -263,6 +317,7 @@ class CultCache: structures, running every check and every user extractor; the cache's state is replaced only once that build finishes, so a refused load admits nothing.""" loaded_values: dict[str, dict[str, Any]] = {} + read_values: dict[tuple[str, str], Any] = {} # (type, key) -> value, one per key, the last read wins; the judge is order-free loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {} loaded_legacy_global_keys: dict[str, str] = {} routes, generic = self._state.stores_by_type, self._state.generic_store @@ -301,9 +356,10 @@ class CultCache: envelope = self._stamped(document, envelope) value = document.decode_payload(envelope.payload) loaded_values.setdefault(envelope.type, {})[envelope.key] = value + read_values[(envelope.type, envelope.key)] = value loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups( - loaded_values, self._state.name_extractors, self._state.index_extractors + [(type, key, value) for (type, key), value in read_values.items()], self._state.name_extractors, self._state.index_extractors ) # Plain attribute assignments: nothing fallible runs once the swap begins. ( @@ -432,12 +488,20 @@ class CultCache: """Validates every record (home and name and index extractors included) before the store is touched; once the store accepts, applying to the cache has no fallible step.""" type = document.type + # A batch naming a key twice lands the last record for it, as C# CultCacheBatch stages one operation per key. The + # judge, the store and the apply all read this one list. + records = list({envelope.key: (envelope, value) for envelope, value in records}.values()) name_extractor = self._state.name_extractors.get(type) index_extractors = self._state.index_extractors.get(type, {}) admitted = [ (envelope, value, self._lookup_keys_for(value, name_extractor, index_extractors)) for envelope, value in records ] + self._refuse_index_sharing( + [(type, envelope.key, keys) for envelope, _, keys in admitted], + self._state.indexes, + {(type, envelope.key) for envelope, _ in records}, + ) store = self._store_for_type(type) legacy_key = self._state.legacy_global_keys.get(type) if store is not None: @@ -552,36 +616,76 @@ class CultCache: name = None if name_extractor is None else extract_value(value, name_extractor) indexes: dict[str, str] = {} for index, extractor in index_extractors.items(): - index_value = extract_value(value, extractor) + index_value = index_text(extract_value(value, extractor)) if index_value is not None: - indexes[index] = str(index_value) + indexes[index] = index_value return _LookupKeys(None if name is None else str(name), indexes) def _derive_lookups( self, - values_by_type: dict[str, dict[str, Any]], + records: list[tuple[str, str, Any]], name_extractors: dict[str, str | Any], index_extractors: dict[str, dict[str, str | Any]], ) -> tuple[_Names, _Indexes, _LookupKeysByType]: - """Builds name and index lookups for the given values in fresh dicts, touching no cache state.""" + """Builds name and index lookups for the given (type, key, value) records, judged whole and order-free, in fresh dicts, touching no cache state.""" names: _Names = {} indexes: _Indexes = {} lookup_keys: _LookupKeysByType = {} - for type, values in values_by_type.items(): - name_extractor = name_extractors.get(type) - type_index_extractors = index_extractors.get(type, {}) - for key, value in values.items(): - keys = self._lookup_keys_for(value, name_extractor, type_index_extractors) - lookup_keys.setdefault(type, {})[key] = keys + landing: list[tuple[str, str, _LookupKeys]] = [] + for type, key, value in records: + keys = self._lookup_keys_for(value, name_extractors.get(type), index_extractors.get(type, {})) + lookup_keys.setdefault(type, {})[key] = keys + landing.append((type, key, keys)) + # Judged whole and order-free, over every type at once and on the file alone (never against what the cache held): a held value can + # have been mutated in place since it was admitted, and another process may have moved a value between records. + self._refuse_index_sharing(landing, {}, ()) + for type in lookup_keys: + for key, keys in lookup_keys.get(type, {}).items(): if keys.name is not None: names[(type, keys.name)] = key for index, index_value in keys.indexes.items(): indexes[(type, index, index_value)] = key return names, indexes, lookup_keys + def _refuse_index_sharing( + self, + landing: list[tuple[str, str, _LookupKeys]], + held: dict[tuple[str, str, str], str], + replaced: Iterable[tuple[str, str]], + ) -> None: + """The one index rule (variants:ruling:unique-indexes): within one type and one unique alias, no two records hold one + non-blank value, judged on the state the admission would leave and never on staging, file or pull order. The holders + of a value are the landing keys projecting it plus the held key unless this admission replaces it. Of every + (type, alias, value) with two or more distinct holders, the refusal names the least by (alias, its least key, its + second-least key, schema name), each compared by UTF-16 code unit, never the value.""" + released = set(replaced) + landed: dict[tuple[str, str, str], list[str]] = {} + for type, key, keys in landing: + for index, index_value in keys.indexes.items(): + landed.setdefault((type, index, index_value), []).append(key) + violations: list[tuple[bytes, bytes, bytes, bytes, str, str, str, str]] = [] + for (type, index, index_value), keys in landed.items(): + held_key = held.get((type, index, index_value)) + if held_key is not None and (type, held_key) not in released: + keys = [*keys, held_key] + distinct = sorted(keys, key=_utf16_order) + if len(distinct) > 1: + document = self._state.documents.get(type) + schema_name = (document.schema_name if document is not None else None) or type + violations.append( + (_utf16_order(index), _utf16_order(distinct[0]), _utf16_order(distinct[1]), _utf16_order(schema_name), + index, distinct[0], distinct[1], schema_name) + ) + if violations: + *_, index, first, second, schema_name = min(violations, key=lambda violation: violation[:4]) + raise CultCacheError( + f"Records {first} and {second} ({schema_name}) both hold unique index '{index}'; " + "an index value names one record." + ) + def _remove_lookups(self, type: str, key: str) -> None: - """Removes the lookups the held record produced at admission, handing a name or index value to - another held record that produced it too. Reads stored lookup keys only; runs no extractor.""" + """Removes the lookups the held record produced at admission, handing a name to another held + record that produced it too (an index value has one holder). Reads stored lookup keys only; runs no extractor.""" held = self._state.lookup_keys.get(type, {}) keys = held.get(key) if keys is None: @@ -596,9 +700,6 @@ class CultCache: if self._state.indexes.get(lookup_key) != key: continue del self._state.indexes[lookup_key] - for candidate_key, candidate in held.items(): - if candidate_key != key and candidate.indexes.get(index) == index_value: - self._state.indexes[lookup_key] = candidate_key def _assert_registered(self, document: DocumentDefinition[Any]) -> None: if self._state.documents.get(document.type) is not document: diff --git a/packages/cultcache-py/src/cultcache_py/documents.py b/packages/cultcache-py/src/cultcache_py/documents.py index 23849713..6101c136 100644 --- a/packages/cultcache-py/src/cultcache_py/documents.py +++ b/packages/cultcache-py/src/cultcache_py/documents.py @@ -157,6 +157,11 @@ class DocumentDefinition(Generic[T]): global_document: bool = False name: str | Extractor[T] | None = None indexes: dict[str, str | Extractor[T]] = field(default_factory=dict) + """Unique lookups (C# `[CultIndex]`): no two records of the type hold one non-blank value, judged at every commit and + load. `get_by_index` answers the one holder.""" + roles: dict[str, str | Extractor[T]] = field(default_factory=dict) + """Selection aliases (C# `[CultRole]`): many records may share a value. Never indexed for lookup, never refuse; the + member's catalog `index_alias` is the same either way.""" payload_encoder: Callable[[Any], bytes] = _json_payload_encoder payload_decoder: Callable[[bytes], Any] = _json_payload_decoder schema_id: str | None = None @@ -190,6 +195,14 @@ class DocumentDefinition(Generic[T]): self.validate(value) return value + def __post_init__(self) -> None: + # Judged here so every constructor passes it: the factories, the dataclass constructor and dataclasses.replace. + for alias in self.roles: + if alias in self.indexes: + raise ValueError( + f"Document type {self.type} declares '{alias}' as both an index and a role; an index already names its role." + ) + def catalog_entry(self) -> CultCacheSchemaCatalogEntry: if self._catalog_entry is not None: return self._catalog_entry @@ -241,6 +254,7 @@ def define_document_type( global_document: bool = False, name: str | Extractor[T] | None = None, indexes: dict[str, str | Extractor[T]] | None = None, + roles: dict[str, str | Extractor[T]] | None = None, payload_encoder: Callable[[Any], bytes] = _json_payload_encoder, payload_decoder: Callable[[bytes], Any] = _json_payload_decoder, schema_id: str | None = None, @@ -259,6 +273,7 @@ def define_document_type( global_document=global_document, name=name, indexes=indexes or {}, + roles=roles or {}, payload_encoder=payload_encoder, payload_decoder=payload_decoder, schema_id=schema_id, @@ -280,6 +295,7 @@ def define_database_entry_type( global_document: bool = False, name: str | Extractor[T] | None = None, indexes: dict[str, str | Extractor[T]] | None = None, + roles: dict[str, str | Extractor[T]] | None = None, schema_id: str | None = None, schema_name: str | None = None, schema_version: str | None = None, @@ -296,7 +312,7 @@ def define_database_entry_type( normalized = _normalize_fields(type, fields) member_name_lookup = name if isinstance(name, str) else None - index_lookup = indexes or {} + index_lookup = {**(roles or {}), **(indexes or {})} def members() -> tuple[CultCacheSchemaCatalogMember, ...]: return tuple( @@ -323,6 +339,7 @@ def define_database_entry_type( global_document=global_document, name=name, indexes=indexes, + roles=roles, payload_encoder=_pack_tree, payload_decoder=_messagepack_payload_decoder, schema_id=schema_id, diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index f5bad365..92892b51 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -103,20 +103,20 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(cache.get_key_by_name(document, "Elixir"), "item:potion") self.assertEqual(cache.get_by_index(document, "category", "Rare").name, "Elixir") - def test_incremental_lookup_delete_restores_duplicate_owner(self) -> None: + def test_incremental_lookup_delete_restores_duplicate_name_owner(self) -> None: document = item_doc() cache = CultCache() cache.register_document_type(document) cache.put(document, "item:first", Item(name="Potion", category="Consumable", value=50)) - cache.put(document, "item:second", Item(name="Potion", category="Consumable", value=60)) + cache.put(document, "item:second", Item(name="Potion", category="Rare", value=60)) self.assertEqual(cache.get_key_by_name(document, "Potion"), "item:second") - self.assertEqual(cache.get_key_by_index(document, "category", "Consumable"), "item:second") cache.delete(document, "item:second") self.assertEqual(cache.get_key_by_name(document, "Potion"), "item:first") self.assertEqual(cache.get_key_by_index(document, "category", "Consumable"), "item:first") + self.assertIsNone(cache.get_key_by_index(document, "category", "Rare")) def test_database_entry_formatter_uses_slot_indexed_messagepack_array(self) -> None: try: @@ -1121,6 +1121,25 @@ class CultCacheTests(unittest.TestCase): catalog_entry=catalog_entry, ) + def test_pulling_a_synthetic_pre_v1_envelope_array_leaves_the_file_bytes_identical(self) -> None: + try: + import msgpack # type: ignore + except ImportError: + self.skipTest("msgpack optional dependency is not installed") + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "legacy.cc" + legacy = msgpack.packb( + [{"key": "note:hello", "type": "note", "payload": {"title": "Hello"}, "storedAt": "2026-10-20T00:00:00Z"}], + use_bin_type=True, + ) + path.write_bytes(legacy) + + envelopes = SingleFileMessagePackBackingStore(path).pull_all() + + self.assertEqual([e.key for e in envelopes], ["note:hello"]) + self.assertEqual(path.read_bytes(), legacy) + self.assertEqual(sorted(p.name for p in Path(tmp).iterdir()), ["legacy.cc"]) + def test_legacy_key_global_loads_without_writing_and_first_write_leaves_one_global(self) -> None: with tempfile.TemporaryDirectory() as tmp: settings = define_document_type("settings", global_document=True) diff --git a/packages/cultcache-py/tests/test_first_call.py b/packages/cultcache-py/tests/test_first_call.py index f8d4b52b..10d6eb27 100644 --- a/packages/cultcache-py/tests/test_first_call.py +++ b/packages/cultcache-py/tests/test_first_call.py @@ -24,7 +24,6 @@ item_doc = define_document_type( encode=lambda item: asdict(item), decode=lambda raw: Item(**raw), name="name", - indexes={"category": "category"}, ) diff --git a/packages/cultcache-py/tests/test_unique_index.py b/packages/cultcache-py/tests/test_unique_index.py new file mode 100644 index 00000000..accc27dd --- /dev/null +++ b/packages/cultcache-py/tests/test_unique_index.py @@ -0,0 +1,664 @@ +from __future__ import annotations + +import itertools +import shutil +import tempfile +import unittest +from dataclasses import asdict, dataclass +from decimal import Decimal +from pathlib import Path + +import msgpack + +from cultcache_py import ( + CultCache, + CultCacheEnvelope, + SingleFileMessagePackBackingStore, + define_database_entry_type, + define_document_type, + leaves as t, + database_entry_field, +) +from cultcache_py.cache import CultCacheError, index_text + +# variants:ruling:unique-indexes. `indexes` are unique within one type, `roles` are selection aliases. The vectors in +# tests/vectors/unique-index are written by the C# reference; the rest of the behaviour is pinned here. +VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "unique-index" + + +@dataclass +class Item: + sku: object = None + serial: object = None + group: object = None + + +def refusal(key_a: str, key_b: str, schema_name: str, alias: str) -> str: + return f"Records {key_a} and {key_b} ({schema_name}) both hold unique index '{alias}'; an index value names one record." + + +def item_doc(type: str = "item", *, indexes=None, roles=None): + return define_document_type( + type, + encode=asdict, + decode=lambda raw: Item(**raw), + indexes={"sku": "sku", "serial": "serial"} if indexes is None else indexes, + roles={"group": "group"} if roles is None else roles, + ) + + +def envelope(document, key: str, item: Item) -> CultCacheEnvelope: + return CultCacheEnvelope.create(key=key, type=document.type, payload=document.encode_payload(item)) + + +def cache_of(*documents, store=None) -> CultCache: + builder = CultCache.builder() + for document in documents: + builder.register_document_type(document) + if store is not None: + builder.add_generic_store(store) + return builder.build() + + +class RecordingStore: + """A store that counts what reaches it; the judge must refuse before any store call.""" + + def __init__(self) -> None: + self.pushed: list[str] = [] + + def pull_all(self): + return [] + + def push(self, envelope) -> None: + self.pushed.append(envelope.key) + + def push_all(self, envelopes) -> None: + self.pushed.extend(envelope.key for envelope in envelopes) + + def delete(self, type, key) -> None: + pass + + +class IndexTextTests(unittest.TestCase): + def test_canonical_text(self) -> None: + cases = [ + ("S-1", "S-1"), + (" a ", " a "), + ("", ""), + ("\x1f", "\x1f"), + ("\x1c", "\x1c"), + ("\x1d", "\x1d"), + ("\x1e", "\x1e"), + ("á Ž", "á Ž"), + ("​", "​"), + (True, "True"), + (False, "False"), + (7, "7"), + (-7, "-7"), + (9223372036854775807, "9223372036854775807"), + (18446744073709551615, "18446744073709551615"), + (0.0, "0"), + (-0.0, "0"), + (2.0, "2"), + (-3.0, "-3"), + (1e21, "1000000000000000000000"), + (0.1, "0.1000000000000000055511151231257827021181583404541015625"), + (-0.5, "-0.5"), + (2.5e-07, "0.00000024999999999999998868702795647156467140348468092270195484161376953125"), + ] + for value, text in cases: + with self.subTest(value=value): + self.assertEqual(index_text(value), text) + self.assertIsInstance(index_text(value), str) + + def test_no_value(self) -> None: + blank = ["", " ", "\t", "\n", "\x0b", "\x0c", "\r", "\x85", "\xa0", " ", "
", "
", " ", + "âŸ", "　", " 　\x85 ", *map(chr, range(0x2000, 0x200B))] + for value in [None, float("nan"), float("inf"), float("-inf"), *blank]: + with self.subTest(value=repr(value)): + self.assertIsNone(index_text(value)) + + def test_blank_set_is_exactly_the_csharp_set(self) -> None: + csharp = {*range(0x09, 0x0E), 0x20, 0x85, 0xA0, 0x1680, *range(0x2000, 0x200B), 0x2028, 0x2029, 0x202F, 0x205F, 0x3000} + self.assertEqual(len(csharp), 25) + got = {code for code in range(0x110000) if 0xD800 > code or code > 0xDFFF if index_text(chr(code)) is None} + self.assertEqual(got, csharp) + # Python's isspace is the wrong test: it also counts U+001C-001F. + self.assertTrue("\x1f".isspace()) + self.assertEqual(index_text("\x1f"), "\x1f") + + def test_floats_render_exactly_not_through_str(self) -> None: + for value in (0.1, 1e-7, 123456789.123456789, 5e-324, 1.7976931348623157e308): + with self.subTest(value=value): + text = index_text(value) + self.assertNotIn("e", text.lower()) + self.assertEqual(Decimal(text), Decimal(value)) + self.assertFalse(text.endswith("0") and "." in text) + + def test_bool_is_not_an_integer(self) -> None: + self.assertEqual(index_text(True), "True") + self.assertNotEqual(index_text(True), "1") + + +class UniqueIndexTests(unittest.TestCase): + def test_a_write_of_a_held_value_at_another_key_is_refused_by_every_write_path(self) -> None: + for alias in ("sku", "serial"): # two unique indexes: each is judged + other = "serial" if alias == "sku" else "sku" + paths = { + "put": lambda cache, d, k, i: cache.put(d, k, i), + "put_envelope": lambda cache, d, k, i: cache.put_envelope(d, envelope(d, k, i)), + "put_envelopes": lambda cache, d, k, i: cache.put_envelopes(d, [envelope(d, k, i)]), + "update": lambda cache, d, k, i: cache.update(d, k, lambda _held: i), + } + for name, write in paths.items(): + with self.subTest(alias=alias, path=name): + document = item_doc() + store = RecordingStore() + cache = cache_of(document, store=store) + cache.put(document, "item:b", Item(sku="CANARY-b", serial="canary-b")) + cache.put(document, "item:a", Item(sku="CANARY-a", serial="canary-a")) + before = cache.snapshot() + pushed = list(store.pushed) + taken = "CANARY-b" if alias == "sku" else "canary-b" + landing = Item(**{alias: taken, other: "fresh"}) + with self.assertRaises(CultCacheError) as caught: + write(cache, document, "item:a", landing) + self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", alias)) + self.assertNotIn("anary", str(caught.exception)) + self.assertEqual(cache.snapshot(), before) + self.assertEqual(store.pushed, pushed) + self.assertEqual(cache.get_key_by_index(document, alias, taken), "item:b") + self.assertIsNone(cache.get_key_by_index(document, other, "fresh")) + + def test_a_record_rewriting_its_own_key_keeps_its_value(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:a", Item(sku="S", serial="1", group="g")) + cache.put(document, "item:a", Item(sku="S", serial="2", group="h")) + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:a") + self.assertIsNone(cache.get_key_by_index(document, "serial", "1")) + self.assertEqual(cache.get_key_by_index(document, "serial", "2"), "item:a") + + def test_the_refusal_names_the_schema_not_the_type(self) -> None: + document = define_document_type( + "t.item", indexes={"sku": "sku"}, schema_name="schema.Item", schema_id="schema.Item.v1" + ) + cache = cache_of(document) + cache.put(document, "k:1", {"sku": "S"}) + with self.assertRaises(CultCacheError) as caught: + cache.put(document, "k:2", {"sku": "S"}) + self.assertEqual(str(caught.exception), refusal("k:1", "k:2", "schema.Item", "sku")) + + def test_the_two_smallest_keys_are_named_in_ordinal_order(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:m", Item(sku="S")) + with self.assertRaises(CultCacheError) as caught: + cache.put_envelopes(document, [envelope(document, "item:z", Item(sku="S")), envelope(document, "item:A", Item(sku="S"))]) + self.assertEqual(str(caught.exception), refusal("item:A", "item:m", "item", "sku")) + + def test_put_envelopes_holding_one_value_twice_is_refused_whole(self) -> None: + document = item_doc() + store = RecordingStore() + cache = cache_of(document, store=store) + with self.assertRaises(CultCacheError) as caught: + cache.put_envelopes( + document, + [ + envelope(document, "item:b", Item(sku="S")), + envelope(document, "item:c", Item(sku="T")), + envelope(document, "item:a", Item(sku="S")), + ], + ) + self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "sku")) + self.assertEqual(cache.snapshot(), {}) + self.assertEqual(store.pushed, []) + self.assertIsNone(cache.get_key_by_index(document, "sku", "T")) + + def test_verdict_does_not_depend_on_batch_order(self) -> None: + document = item_doc() + rows = [("item:a", Item(sku="S", serial="1")), ("item:b", Item(sku="T", serial="1")), ("item:c", Item(sku="U"))] + for order in itertools.permutations(rows): + with self.subTest(order=[key for key, _ in order]): + cache = cache_of(document) + with self.assertRaises(CultCacheError) as caught: + cache.put_envelopes(document, [envelope(document, key, item) for key, item in order]) + self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "serial")) + self.assertEqual(cache.snapshot(), {}) + + def test_put_envelopes_swapping_two_records_values_lands(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:a", Item(sku="S", serial="1")) + cache.put(document, "item:b", Item(sku="T", serial="2")) + cache.put_envelopes( + document, + [envelope(document, "item:a", Item(sku="T", serial="2")), envelope(document, "item:b", Item(sku="S", serial="1"))], + ) + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b") + self.assertEqual(cache.get_key_by_index(document, "sku", "T"), "item:a") + self.assertEqual(cache.get_key_by_index(document, "serial", "1"), "item:b") + self.assertEqual(cache.get_key_by_index(document, "serial", "2"), "item:a") + + def test_a_batch_may_take_a_value_the_batch_releases(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:a", Item(sku="S")) + cache.put_envelopes(document, [envelope(document, "item:a", Item(sku="T")), envelope(document, "item:b", Item(sku="S"))]) + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b") + self.assertEqual(cache.get_key_by_index(document, "sku", "T"), "item:a") + + def test_delete_frees_the_value(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:a", Item(sku="S")) + cache.delete(document, "item:a") + self.assertIsNone(cache.get_key_by_index(document, "sku", "S")) + cache.put(document, "item:b", Item(sku="S")) + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b") + + def test_blank_values_never_conflict(self) -> None: + document = item_doc() + cache = cache_of(document) + blanks = ["", " ", "\x85", "　", None] + for number, blank in enumerate(blanks): + cache.put(document, f"item:blank{number}", Item(sku=blank, serial=blank)) + cache.put_envelopes(document, [envelope(document, "item:more", Item(sku=" ")), envelope(document, "item:more2", Item(sku=""))]) + for blank in ("", " ", "\x85"): + self.assertIsNone(cache.get_key_by_index(document, "sku", blank)) + # U+001F and U+FEFF are values. + cache.put(document, "item:us", Item(sku="\x1f")) + cache.put(document, "item:bom", Item(sku="")) + self.assertEqual(cache.get_key_by_index(document, "sku", "\x1f"), "item:us") + self.assertEqual(cache.get_key_by_index(document, "sku", ""), "item:bom") + with self.assertRaises(CultCacheError): + cache.put(document, "item:us2", Item(sku="\x1f")) + + def test_values_compare_as_the_canonical_text(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:a", Item(sku=1)) + for same in (1, 1.0, "1"): + with self.subTest(same=same), self.assertRaises(CultCacheError): + cache.put(document, "item:b", Item(sku=same)) + cache.put(document, "item:true", Item(sku=True)) + self.assertEqual(cache.get_key_by_index(document, "sku", "True"), "item:true") + self.assertEqual(cache.get_key_by_index(document, "sku", "1"), "item:a") + cache.put(document, "item:f", Item(sku=0.1)) + self.assertEqual(cache.get_key_by_index(document, "sku", "0.1000000000000000055511151231257827021181583404541015625"), "item:f") + + def test_two_types_may_share_an_alias_and_a_value(self) -> None: + one, two = item_doc("one"), item_doc("two") + cache = cache_of(one, two) + cache.put(one, "k:1", Item(sku="S", serial="1")) + cache.put(two, "k:2", Item(sku="S", serial="1")) + self.assertEqual(cache.get_key_by_index(one, "sku", "S"), "k:1") + self.assertEqual(cache.get_key_by_index(two, "sku", "S"), "k:2") + cache.delete(one, "k:1") + self.assertEqual(cache.get_key_by_index(two, "sku", "S"), "k:2") + + def test_roles_may_share_values_and_are_not_looked_up(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:a", Item(sku="1", group="g")) + cache.put(document, "item:b", Item(sku="2", group="g")) + self.assertIsNone(cache.get_key_by_index(document, "group", "g")) + self.assertIsNone(cache.get_by_index(document, "group", "g")) + + def test_an_alias_in_both_indexes_and_roles_is_refused(self) -> None: + with self.assertRaisesRegex(ValueError, "both an index and a role"): + define_document_type("bad", indexes={"a": "a"}, roles={"a": "a"}) + with self.assertRaisesRegex(ValueError, "both an index and a role"): + define_database_entry_type("bad", [("a", 0)], indexes={"a": "a"}, roles={"a": "a"}) + define_document_type("fine", indexes={"a": "a"}, roles={"b": "a"}) + + def test_register_index_over_held_duplicates_installs_nothing(self) -> None: + document = item_doc(indexes={}, roles={}) + cache = cache_of(document) + cache.put(document, "item:b", Item(sku="S", serial="1")) + cache.put(document, "item:a", Item(sku="S", serial="2")) + with self.assertRaises(CultCacheError) as caught: + cache.register_index(document, "sku", "sku") + self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "sku")) + self.assertIsNone(cache.get_key_by_index(document, "sku", "S")) + cache.register_index(document, "serial", "serial") # an installed alias is not poisoned by the refused one + self.assertEqual(cache.get_key_by_index(document, "serial", "2"), "item:a") + # Nothing was installed, so a write sharing the refused alias's value is still admitted. + cache.put(document, "item:c", Item(sku="S", serial="3")) + + def test_registration_names_an_installed_alias_a_caller_mutated_into_a_collision(self) -> None: + document = item_doc(indexes={"sku": "sku"}, roles={}) + cache = cache_of(document) + cache.put(document, "item:x", Item(sku="1", serial="s1")) + cache.put(document, "item:y", Item(sku="2", serial="s2")) + cache.get_required(document, "item:y").sku = "1" # the cache hands out its held value + with self.assertRaises(CultCacheError) as caught: + cache.register_index(document, "serial", "serial") + self.assertEqual(str(caught.exception), refusal("item:x", "item:y", "item", "sku")) + + def test_derive_names_the_ordinal_first_alias_then_value(self) -> None: + document = item_doc(indexes={}, roles={}) + cache = cache_of(document) + cache.put(document, "item:1", Item(sku="a", serial="p", group="x")) + cache.put(document, "item:2", Item(sku="a", serial="p", group="x")) + cache.put(document, "item:3", Item(sku="b", serial="q", group="y")) + cache.put(document, "item:4", Item(sku="b", serial="q", group="y")) + with self.assertRaises(CultCacheError) as caught: + cache.register_index(document, "serial", "serial") + self.assertEqual(str(caught.exception), refusal("item:1", "item:2", "item", "serial")) + + def test_a_document_in_two_unique_indexes_loads_and_registers_through_every_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.cc" + document = item_doc() + cache = cache_of(document, store=SingleFileMessagePackBackingStore(path)) + cache.put(document, "item:a", Item(sku="S", serial="1")) + cache.put(document, "item:b", Item(sku="T", serial="2")) + # a store written by a looser definition, then opened by this one + loose = item_doc(indexes={}, roles={}) + writer = cache_of(loose, store=SingleFileMessagePackBackingStore(path)) + writer.pull_all_backing_stores() + writer.put(loose, "item:c", Item(sku="U", serial="2")) + before = path.read_bytes() + reader = cache_of(document, store=SingleFileMessagePackBackingStore(path)) + with self.assertRaises(CultCacheError) as caught: + reader.pull_all_backing_stores() + self.assertEqual(str(caught.exception), refusal("item:b", "item:c", "item", "serial")) + self.assertEqual(path.read_bytes(), before) + self.assertEqual(reader.snapshot(), {}) + + def test_a_refused_load_keeps_the_held_state(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "items.cc" + loose = item_doc(indexes={}, roles={}) + writer = cache_of(loose, store=SingleFileMessagePackBackingStore(path)) + writer.put(loose, "item:a", Item(sku="S")) + writer.put(loose, "item:b", Item(sku="S")) + document = item_doc() + cache = cache_of(document, store=SingleFileMessagePackBackingStore(path)) + with self.assertRaises(CultCacheError): + cache.pull_all_backing_stores() + self.assertIsNone(cache.get_key_by_index(document, "sku", "S")) + self.assertEqual(cache.get_all(document), []) + + +class SimulationFactTests(unittest.TestCase): + def test_simulation_fact_catalog_entry_is_unchanged_by_indexes_to_roles(self) -> None: + import re + + from cultnet_py.cultmesh_contracts import SIMULATION_FACT_DOCUMENT_TYPE, simulation_fact_document as document + + aliases = {"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"} + self.assertEqual(document.roles, aliases) + self.assertEqual(document.indexes, {}) + entry = document.catalog_entry() + snake = lambda name: re.sub(r"(? None: + from cultnet_py.cultmesh_contracts import CultMeshSimulationFact, simulation_fact_document as document + + cache = cache_of(document) + for number in range(3): + fact = CultMeshSimulationFact.from_candidate( + {"shardId": "s", "shardEpoch": 1, "frame": 4, "subjectId": "x", "claimKind": "k", "claimHash": f"h{number}", + "witnessCount": 1, "supportWeight": 1.0, "totalWeight": 1.0, "confidence": 1.0} + ) + cache.put(document, f"simulation:{number}", fact) + self.assertEqual(len(cache.get_all(document)), 3) + self.assertIsNone(cache.get_key_by_index(document, "shard", "s")) + + +class VectorTests(unittest.TestCase): + def test_values_rows(self) -> None: + document = item_doc(indexes={"sku": "sku"}, roles={}) + for label, kind, value, text in msgpack.unpackb((VECTORS / "values.msgpack").read_bytes(), raw=False): + with self.subTest(label=label): + self.assertEqual(index_text(value), text) + cache = cache_of(document) + cache.put(document, "item:1", Item(sku=value)) + if text is None: + cache.put(document, "item:2", Item(sku=value)) # blank: two records commit + self.assertEqual(len(cache.get_all(document)), 2) + else: + self.assertEqual(cache.get_key_by_index(document, "sku", text), "item:1") + with self.assertRaises(CultCacheError): + cache.put(document, "item:2", Item(sku=value)) + + def test_store_verdicts(self) -> None: + unit = define_database_entry_type + outcomes = msgpack.unpackb((VECTORS / "outcomes.msgpack").read_bytes(), raw=False) + self.assertEqual(len(outcomes), 3) + for name, alias, verdict, key_a, key_b in outcomes: + for roles_first in (False, True): # the verdict does not depend on declaration order + with self.subTest(file=name, alias=alias, roles_first=roles_first), tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / name + shutil.copyfile(VECTORS / name, path) + before = path.read_bytes() + fields = [database_entry_field("sku", 0, codec=t.string), database_entry_field("category", 1, codec=t.string)] + declared = {"indexes": {alias: "sku"} if alias else {}, "roles": {} if alias else {"Sku": "sku"}} + if roles_first: + declared = dict(reversed(list(declared.items()))) + document = unit( + "tests.unique_item", + fields, + schema_id="sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", + schema_name="tests.unique_item", + **declared, + ) + cache = cache_of(document, store=SingleFileMessagePackBackingStore(path)) + if verdict == "open": + cache.pull_all_backing_stores() + self.assertGreaterEqual(len(cache.get_all(document)), 2) + else: + with self.assertRaises(CultCacheError) as caught: + cache.pull_all_backing_stores() + self.assertEqual(str(caught.exception), refusal(key_a, key_b, "tests.unique_item", alias)) + self.assertNotIn("S-1", str(caught.exception)) + self.assertEqual(cache.snapshot(), {}) + self.assertEqual(path.read_bytes(), before) + + + def test_refusals(self) -> None: + ids = { + "tests.unique_item": "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", + "tests.uq_pair": "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", + "tests.uq_alias": "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", + "tests.uq_wide": "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", + "tests.uq_alpha": "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", + "tests.uq_beta": "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", + } + aliases = { + "tests.uq_pair": {"Sku": "sku", "Alt": "second"}, + "tests.uq_alias": {"Sku": "sku", "alt": "second"}, + "tests.uq_wide": {"\uff21": "sku", "\U00010400": "second"}, # U+FF21 and U+10400: code-unit order puts the astral alias first + } + documents = { + name: define_database_entry_type( + name, + [database_entry_field("sku", 0, codec=t.string), database_entry_field("second", 1, codec=t.string)], + indexes=aliases.get(name, {"Sku": "sku"}), + roles={}, + schema_id=schema_id, + schema_name=name, + ) + for name, schema_id in ids.items() + } + rows = msgpack.unpackb((VECTORS / "refusals.msgpack").read_bytes(), raw=False) + walked = 0 + for label, ops, expected in rows: + if any(op[0] == "commit" and len({schema for _, schema, _, _ in op[1]}) > 1 for op in ops): + continue # put_envelopes takes one type: no batch across types (rows 3 and 26) + with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "store.cc" + state: dict[str, CultCache] = {} + + def perform(op) -> None: + if op[0] == "open": + if op[1] is not None: + shutil.copyfile(VECTORS / op[1], path) + state["cache"] = cache_of(*documents.values(), store=SingleFileMessagePackBackingStore(path)) + state["cache"].pull_all_backing_stores() + elif op[0] == "pull": + shutil.copyfile(VECTORS / op[1], path) + state["cache"].pull_all_backing_stores() + else: + document = documents[op[1][0][1]] + state["cache"].put_envelopes( + document, [envelope(document, key, {"sku": sku, "second": second}) for key, _, sku, second in op[1]] + ) + + for op in ops[:-1]: + perform(op) + if expected is None: + perform(ops[-1]) + else: + last = ops[-1] + if last[0] == "open": + before = (VECTORS / last[1]).read_bytes() + elif last[0] == "pull": + before = (VECTORS / last[1]).read_bytes() + else: + before = path.read_bytes() if path.exists() else b"" + k1, k2, schema, alias = expected + with self.assertRaises(CultCacheError) as caught: + perform(last) + self.assertEqual(str(caught.exception), refusal(k1, k2, schema, alias)) + self.assertEqual(path.read_bytes() if path.exists() else b"", before) + walked += 1 + self.assertEqual(walked, 35) + + +class ListStore(RecordingStore): + """A store whose contents the test swaps between loads: another process rewrote the file.""" + + def __init__(self, envelopes=()) -> None: + super().__init__() + self.envelopes = list(envelopes) + + def pull_all(self): + return list(self.envelopes) + + +class BatchAndOrderTests(unittest.TestCase): + def test_a_batch_naming_a_key_twice_lands_the_last_record_and_keeps_every_held_lookup(self) -> None: + # C# CultCacheBatch stages one operation per key, the last one named. + document = item_doc() + store = RecordingStore() + cache = cache_of(document, store=store) + cache.put(document, "item:h", Item(sku="S")) + store.pushed.clear() + cache.put_envelopes( + document, [envelope(document, "item:k", Item(sku="S")), envelope(document, "item:k", Item(sku="T"))] + ) + self.assertEqual(store.pushed, ["item:k"]) + self.assertEqual(cache.get_required(document, "item:k").sku, "T") + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:h") + self.assertEqual(cache.get_key_by_index(document, "sku", "T"), "item:k") + with self.assertRaises(CultCacheError) as caught: + cache.put(document, "item:z", Item(sku="S")) + self.assertEqual(str(caught.exception), refusal("item:h", "item:z", "item", "sku")) + + def test_a_batch_naming_a_key_twice_is_judged_on_the_record_that_lands(self) -> None: + document = item_doc() + cache = cache_of(document) + cache.put(document, "item:h", Item(sku="S")) + # the first record for k would collide with h; only the last lands, so the batch is admitted + cache.put_envelopes( + document, [envelope(document, "item:k", Item(sku="S")), envelope(document, "item:k", Item(sku="T"))] + ) + # and the reverse order lands the colliding record, which is refused + with self.assertRaises(CultCacheError): + cache.put_envelopes( + document, [envelope(document, "item:j", Item(sku="U")), envelope(document, "item:j", Item(sku="S"))] + ) + self.assertNotIn("item:j", cache.snapshot().get("item", {})) + + def test_a_load_judges_every_type_at_once_aliases_in_ordinal_order(self) -> None: + first = item_doc("first", indexes={"zeta": "sku"}, roles={}) + second = item_doc("second", indexes={"alpha": "sku"}, roles={}) + loose_first, loose_second = item_doc("first", indexes={}, roles={}), item_doc("second", indexes={}, roles={}) + store = ListStore( + [envelope(loose_first, "f:1", Item(sku="S")), envelope(loose_first, "f:2", Item(sku="S"))] + + [envelope(loose_second, "s:1", Item(sku="S")), envelope(loose_second, "s:2", Item(sku="S"))] + ) + cache = cache_of(first, second, store=store) + with self.assertRaises(CultCacheError) as caught: + cache.pull_all_backing_stores() + self.assertEqual(str(caught.exception), refusal("s:1", "s:2", "second", "alpha")) + + def test_aliases_sort_by_utf16_code_unit_not_code_point(self) -> None: + astral, bmp = "k\U0001F600", "k\uff01" # code points: bmp < astral; UTF-16 units: D83D < FF01 + document = item_doc(indexes={bmp: "sku", astral: "serial"}, roles={}) + cache = cache_of(document) + rows = [Item(sku="S", serial="T"), Item(sku="S", serial="T")] + with self.assertRaises(CultCacheError) as caught: + cache.put_envelopes(document, [envelope(document, f"item:{n}", row) for n, row in enumerate(rows)]) + self.assertEqual(str(caught.exception), refusal("item:0", "item:1", "item", astral)) + + def test_a_load_names_the_least_violation_whatever_order_the_store_yields(self) -> None: + alpha, beta = item_doc("alpha", indexes={"k": "sku"}, roles={}), item_doc("beta", indexes={"k": "sku"}, roles={}) + loose_alpha, loose_beta = item_doc("alpha", indexes={}, roles={}), item_doc("beta", indexes={}, roles={}) + store = ListStore( + [ + envelope(loose_alpha, "a:0", Item(sku="Q")), + envelope(loose_beta, "b:1", Item(sku="S")), + envelope(loose_beta, "b:2", Item(sku="S")), + envelope(loose_alpha, "a:1", Item(sku="P")), + envelope(loose_alpha, "a:2", Item(sku="P")), + ] + ) + cache = cache_of(alpha, beta, store=store) + with self.assertRaises(CultCacheError) as caught: + cache.pull_all_backing_stores() + self.assertEqual(str(caught.exception), refusal("a:1", "a:2", "alpha", "k")) + + def test_register_index_refuses_an_alias_the_definition_declares_as_a_role(self) -> None: + document = item_doc() + cache = cache_of(document) + with self.assertRaises(CultCacheError): + cache.register_index(document, "group", "group") + cache.put_envelopes( + document, [envelope(document, "item:1", Item(group="guide")), envelope(document, "item:2", Item(group="guide"))] + ) + + +class LoadFreshnessTests(unittest.TestCase): + def test_a_load_is_judged_on_the_file_alone_not_against_the_held_map(self) -> None: + document = item_doc() + store = ListStore([envelope(document, "item:b", Item(sku="S")), envelope(document, "item:c", Item(sku="T"))]) + cache = cache_of(document, store=store) + cache.pull_all_backing_stores() + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b") + # another process moved S from b to a, and b to T's old place + store.envelopes = [envelope(document, "item:a", Item(sku="S")), envelope(document, "item:b", Item(sku="U")), store.envelopes[1]] + cache.pull_all_backing_stores() + self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:a") + self.assertEqual(cache.get_key_by_index(document, "sku", "U"), "item:b") + + +class DefinitionTests(unittest.TestCase): + def test_every_constructor_refuses_an_alias_in_both_indexes_and_roles(self) -> None: + from dataclasses import replace + + from cultcache_py import DocumentDefinition + + with self.assertRaisesRegex(ValueError, "both an index and a role"): + DocumentDefinition(type="item", indexes={"sku": "sku"}, roles={"sku": "sku"}) + with self.assertRaisesRegex(ValueError, "both an index and a role"): + replace(item_doc(roles={}), roles={"sku": "sku"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultcache-rs/CHANGELOG.md b/packages/cultcache-rs/CHANGELOG.md index c417d535..15011491 100644 --- a/packages/cultcache-rs/CHANGELOG.md +++ b/packages/cultcache-rs/CHANGELOG.md @@ -8,12 +8,16 @@ only in git history. ### Breaking -- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. - A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3). +- `#[cultcache(index)]` means unique, as in C# `RefuseIndexSharing`: within one type and one alias no two records hold one non-blank value. A put, batch, push or load that would make two records hold a value is refused, naming both keys and the alias, and a store that already holds a duplicate does not open. A grouping index is no longer expressible; a consumer that relied on shared values must drop the `index` and look up by another way. +- `bool` index and lookup text renders `True` / `False`, as C# does, where it rendered `true` / `false`. +- `DatabaseEntry::UNIQUE_INDEXES` replaces `cult_indexes` as the one index declaration and changes type (its members are the unique aliases); a hand implementation overriding `cult_indexes` no longer compiles. +- `register_entry_type` over a type that already holds records re-judges them under the new declaration (unique indexes and the global rule) and refuses, changing nothing, when they violate it. A declaration with an index also refuses when a held record cannot be decoded as the new type; a declaration without one does not read the records, so it does not. ### Added -- Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name or value find the one with the smallest key. A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`). +- Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name find the one with the smallest key (an index value is unique, below). A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`). - `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::()`. ## [0.3.1] diff --git a/packages/cultcache-rs/README.md b/packages/cultcache-rs/README.md index 2c5c8027..45724843 100644 --- a/packages/cultcache-rs/README.md +++ b/packages/cultcache-rs/README.md @@ -394,6 +394,10 @@ fn outcome(error: &anyhow::Error) { } ``` +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-rs from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) + ## Near-Term Ergonomic Improvements 1. **Derive macro** @@ -412,7 +416,9 @@ fn outcome(error: &anyhow::Error) { 3. **Name and index lookups** - `cache.get_by_name::("Potion")` - - `cache.get_by_index::("faction", "Lucent")` + - `cache.get_by_index::("callsign", "Lucent-7")`: `#[cultcache(index)]` is unique. Within one type no two + records hold one non-blank value of the alias; a put, batch or load that would make two is refused, naming both + keys and the alias, and writes nothing. A grouping (faction, kind, status) is not an index: filter `get_all`. - generated extractors instead of reflection strings 4. **Schema/version metadata** diff --git a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs index fe4e57c6..c7f9a3a2 100644 --- a/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs +++ b/packages/cultcache-rs/cultcache-rs-derive/src/lib.rs @@ -69,23 +69,16 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result = slots.iter().filter_map(|slot| slot.index.as_ref().map(|alias| (alias, &slot.ident))).collect(); + let index_rows = index_members.iter().map(|(alias, member)| { quote! { - if let ::std::option::Option::Some(value) = ::cultcache_rs::CultLookupValue::lookup_value(&self.#member) { - held.push((#alias, value)); - } + (#alias, (|record: &Self| ::cultcache_rs::CultLookupValue::lookup_value(&record.#member)) + as fn(&Self) -> ::std::option::Option<::std::string::String>) } }); - let index_pushes: Vec<_> = index_pushes.collect(); - let indexes_fn = (!index_pushes.is_empty()).then(|| { - quote! { - fn cult_indexes(&self) -> ::std::vec::Vec<(&'static str, ::std::string::String)> { - let mut held = ::std::vec::Vec::new(); - #(#index_pushes)* - held - } - } + let index_rows: Vec<_> = index_rows.collect(); + let unique_indexes = (!index_rows.is_empty()).then(|| { + quote! { const UNIQUE_INDEXES: &'static [::cultcache_rs::CultIndexMember] = &[#(#index_rows),*]; } }); Ok(quote! { @@ -93,8 +86,8 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result, } @@ -941,15 +935,15 @@ mod tests { ); assert!(tokens.contains("constGLOBAL:bool=true"), "{tokens}"); assert!(tokens.contains("fncult_name"), "{tokens}"); - assert!(tokens.contains("(\"code\",value)"), "{tokens}"); - assert!(tokens.contains("(\"sku\",value)"), "{tokens}"); + assert!(tokens.contains("(\"code\","), "{tokens}"); + assert!(tokens.contains("(\"sku\","), "{tokens}"); } #[test] fn a_record_without_lookups_generates_none_of_them() { let tokens = entry_tokens(r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0)] a: String }"#); assert!(tokens.contains("constGLOBAL:bool=false"), "{tokens}"); - assert!(!tokens.contains("fncult_name") && !tokens.contains("fncult_indexes"), "{tokens}"); + assert!(!tokens.contains("fncult_name") && !tokens.contains("UNIQUE_INDEXES"), "{tokens}"); } // ResolveIndexAlias: a blank alias is the member's name. @@ -958,7 +952,7 @@ mod tests { let tokens = entry_tokens( r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = " ")] code: String }"#, ); - assert!(tokens.contains("(\"code\",value)"), "{tokens}"); + assert!(tokens.contains("(\"code\","), "{tokens}"); } #[test] diff --git a/packages/cultcache-rs/src/lib.rs b/packages/cultcache-rs/src/lib.rs index 4ceb43f2..e5e88b9c 100644 --- a/packages/cultcache-rs/src/lib.rs +++ b/packages/cultcache-rs/src/lib.rs @@ -168,12 +168,14 @@ pub trait DatabaseEntry: Serialize + DeserializeOwned + Clone + Send + 'static { fn cult_name(&self) -> Option { None } - /// Each `#[cultcache(index)]` member as (alias, value), which `get_by_index` finds the record by. - fn cult_indexes(&self) -> Vec<(&'static str, String)> { - Vec::new() - } + /// The `#[cultcache(index)]` members, the one declaration: each is an alias with the reader of its value. Each is + /// unique: within one type no two records hold one non-blank value of it, and `get_by_index` finds the record by it. + const UNIQUE_INDEXES: &'static [CultIndexMember] = &[]; } +/// An index alias and how to read its value from a record; `None` is no value. +pub type CultIndexMember = (&'static str, fn(&T) -> Option); + /// What a `#[cultcache(name)]` or `#[cultcache(index)]` member says it is called: its text, or `None` for a member /// that holds no value (an `Option` that is `None`). The cache treats an empty name and a blank index value as none. pub trait CultLookupValue { @@ -192,6 +194,13 @@ impl CultLookupValue for Option { } } +/// C# renders a boolean `True` or `False`; the index text is shared across runtimes. +impl CultLookupValue for bool { + fn lookup_value(&self) -> Option { + Some(if *self { "True" } else { "False" }.to_string()) + } +} + macro_rules! lookup_value_by_display { ($($ty:ty),* $(,)?) => {$( impl CultLookupValue for $ty { @@ -202,7 +211,7 @@ macro_rules! lookup_value_by_display { )*}; } -lookup_value_by_display!(bool, char, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); +lookup_value_by_display!(char, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); /// A global type already holds a record under another key; a second would leave `get_global` with two answers. #[derive(Debug, Clone, PartialEq, Eq)] @@ -2719,9 +2728,56 @@ pub struct CultCache { globals: BTreeSet, schema_name_definitions: BTreeMap, entries: BTreeMap<(String, String), CultCacheEnvelope>, + /// Derived from `entries` by the uniqueness judge: (type, alias, value) -> the one key holding that value. It + /// decides nothing; `get_by_index` reads it. + index_holders: IndexHolders, + /// Per registered type with `UNIQUE_INDEXES`: decodes a payload as that type and returns its non-blank index + /// values. Registration data, so the judge decodes only indexed types. + projectors: BTreeMap, stores: Vec, } +type IndexProjector = fn(&[u8]) -> Result>; +/// Orders by UTF-16 code unit (C# `StringComparer.Ordinal`), not by UTF-8 bytes or code point. +fn utf16_cmp(left: &str, right: &str) -> std::cmp::Ordering { + left.encode_utf16().cmp(right.encode_utf16()) +} + +/// Orders violations (alias, least key, second key, schema) field by field, each by UTF-16. +fn violation_cmp(left: &[&str; 4], right: &[&str; 4]) -> std::cmp::Ordering { + left.iter().zip(right).map(|(a, b)| utf16_cmp(a, b)).find(|order| order.is_ne()).unwrap_or(std::cmp::Ordering::Equal) +} + +type IndexSlot = (String, String, String); +type IndexHolders = BTreeMap; + +fn project_indexes(payload: &[u8]) -> Result> { + let document = decode_payload::(payload)?; + Ok(T::UNIQUE_INDEXES + .iter() + .filter_map(|(alias, read)| read(&document).map(|value| (*alias, value))) + .filter(|(_, value)| !value.trim().is_empty()) + .collect()) +} + +/// What an admission does to the holder map: slots it releases (still held by the key that held them) and slots it takes. +#[derive(Default)] +struct IndexChange { + released: Vec<(IndexSlot, String)>, + taken: Vec<(IndexSlot, String)>, +} + +impl IndexChange { + fn apply(self, holders: &mut IndexHolders) { + for (slot, key) in self.released { + if holders.get(&slot) == Some(&key) { + holders.remove(&slot); + } + } + holders.extend(self.taken); + } +} + impl CultCache { pub fn new() -> Self { Self { @@ -2729,6 +2785,8 @@ impl CultCache { globals: BTreeSet::new(), schema_name_definitions: BTreeMap::new(), entries: BTreeMap::new(), + index_holders: BTreeMap::new(), + projectors: BTreeMap::new(), stores: Vec::new(), } } @@ -2756,12 +2814,40 @@ impl CultCache { existing_type )); } - self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME); + // The declaration decides what the held records hold (index projector, global flag), so a changed declaration + // re-judges them; a held violation (or a record the new type cannot decode) refuses the registration and + // changes nothing. + let previous = self.projectors.get(T::TYPE).copied(); + let was_global = self.globals.contains(T::TYPE); + if T::UNIQUE_INDEXES.is_empty() { + self.projectors.remove(T::TYPE); + } else { + self.projectors.insert(T::TYPE.to_string(), project_indexes::); + } if T::GLOBAL { self.globals.insert(T::TYPE.to_string()); } else { self.globals.remove(T::TYPE); } + let held_type = self.entries.range((T::TYPE.to_string(), String::new())..).next().is_some_and(|((held, _), _)| held == T::TYPE); + if held_type { + match self.refuse_second_globals(&self.entries).and_then(|()| self.judge_image(&self.entries)) { + Ok(holders) => self.index_holders = holders, + Err(refusal) => { + match previous { + Some(project) => self.projectors.insert(T::TYPE.to_string(), project), + None => self.projectors.remove(T::TYPE), + }; + if was_global { + self.globals.insert(T::TYPE.to_string()); + } else { + self.globals.remove(T::TYPE); + } + return Err(refusal); + } + } + } + self.definitions.insert(T::TYPE.to_string(), T::SCHEMA_NAME); self.schema_name_definitions .insert(T::SCHEMA_NAME.to_string(), T::TYPE.to_string()); Ok(()) @@ -2869,7 +2955,9 @@ impl CultCache { } } self.refuse_second_globals(&loaded)?; + let holders = self.judge_image(&loaded)?; self.entries = loaded; + self.index_holders = holders; Ok(()) } @@ -2973,20 +3061,18 @@ impl CultCache { self.find_record::(|document| document.cult_name().as_deref() == Some(name)) } - /// The record whose `#[cultcache(index)]` member `alias` holds `value`. A blank value is not indexed, so it finds - /// nothing. Records of one type sharing a value find the one with the smallest key (ordinal), whatever order they - /// were written or loaded in; removing it finds the next. + /// The record whose `#[cultcache(index)]` member `alias` holds `value`: at most one, since an index is unique. A + /// blank value is not indexed, so it finds nothing; neither does an alias `T` does not declare. pub fn get_by_index(&self, alias: &str, value: &str) -> Result> { self.require_entry_type::()?; - if value.trim().is_empty() { + if value.trim().is_empty() || !T::UNIQUE_INDEXES.iter().any(|(declared, _)| *declared == alias) { return Ok(None); } - self.find_record::(|document| { - document - .cult_indexes() - .iter() - .any(|(held_alias, held_value)| *held_alias == alias && held_value == value) - }) + let slot = (T::TYPE.to_string(), alias.to_string(), value.to_string()); + match self.index_holders.get(&slot) { + Some(key) => self.get::(key), + None => Ok(None), + } } /// The first record of `T`, in key order, that `matches`. @@ -3029,10 +3115,12 @@ impl CultCache { pub fn put(&mut self, key: impl Into, value: &T) -> Result { let (entry, parsed) = self.prepare_entry(key, value)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; + let change = self.judge_unique(&[&entry])?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.push(&entry)?; } self.entries.insert(entry_id(&entry), entry); + change.apply(&mut self.index_holders); Ok(parsed) } @@ -3141,6 +3229,7 @@ impl CultCache { next.insert(entry_id(entry), entry.clone()); } self.refuse_second_globals(&next)?; + let change = self.judge_unique(&staged.iter().collect::>())?; if let Some(store_index) = store_index { // The home store receives its own records only: those it already // holds in this cache plus the staged ones. @@ -3154,16 +3243,19 @@ impl CultCache { .push_all(&snapshot, PushAllOptions::default())?; } self.entries = next; + change.apply(&mut self.index_holders); Ok(()) } pub fn put_envelope(&mut self, entry: CultCacheEnvelope) -> Result { let parsed = self.validate_envelope::(&entry)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; + let change = self.judge_unique(&[&entry])?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.push(&entry)?; } self.entries.insert(entry_id(&entry), entry); + change.apply(&mut self.index_holders); Ok(parsed) } @@ -3193,10 +3285,12 @@ impl CultCache { } require_payload_depth(&entry)?; self.refuse_second_global(&self.entries, &entry.r#type, &entry.key)?; + let change = self.judge_unique(&[&entry])?; if let Some(home) = self.home_index(&entry.r#type)? { self.stores[home].store.push(&entry)?; } self.entries.insert(entry_id(&entry), entry); + change.apply(&mut self.index_holders); Ok(()) } @@ -3247,10 +3341,12 @@ impl CultCache { let Some(entry) = self.entries.get(&id).cloned() else { return Ok(false); }; + let released = self.projected_slots(&entry)?; if let Some(home) = self.home_index(T::TYPE)? { self.stores[home].store.delete(&entry)?; } self.entries.remove(&id); + IndexChange { released, taken: Vec::new() }.apply(&mut self.index_holders); Ok(true) } @@ -3279,10 +3375,100 @@ impl CultCache { loaded.insert(entry_id(&entry), entry); } self.refuse_second_globals(&loaded)?; + let holders = self.judge_image(&loaded)?; self.entries = loaded; + self.index_holders = holders; Ok(()) } + /// The slots an envelope holds: its type's non-blank index values, each held by its key. + fn projected_slots(&self, entry: &CultCacheEnvelope) -> Result> { + let Some(project) = self.projectors.get(&entry.r#type) else { + return Ok(Vec::new()); + }; + let held = project(&entry.payload).with_context(|| { + format!("failed to project the unique indexes of CultCache entry {:?} at key {:?}", entry.r#type, entry.key) + })?; + Ok(held + .into_iter() + .map(|(alias, value)| ((entry.r#type.clone(), alias.to_string(), value), entry.key.clone())) + .collect()) + } + + /// The one index rule (C# `RefuseIndexSharing`), judged on the result of an admission: within one type and one + /// unique alias, no two records hold one non-blank value. The holders of a value are the held key (unless this + /// admission rewrites it) plus every landing key projecting it; two distinct holders refuse, naming both keys + /// (ordinal) and the alias, never the value. A key landing twice lands as its last envelope. + fn judge_unique(&self, landing: &[&CultCacheEnvelope]) -> Result { + self.judge_against(&self.index_holders, landing, true) + } + + /// Judges a whole image against an empty holder map and returns the map it installs. + fn judge_image(&self, image: &BTreeMap<(String, String), CultCacheEnvelope>) -> Result { + let landing: Vec<_> = image.values().collect(); + let mut holders = IndexHolders::new(); + self.judge_against(&holders, &landing, false)?.apply(&mut holders); + Ok(holders) + } + + fn judge_against(&self, held: &IndexHolders, landing: &[&CultCacheEnvelope], replacing: bool) -> Result { + let mut last: BTreeMap<(&str, &str), &CultCacheEnvelope> = BTreeMap::new(); + for entry in landing { + last.insert((entry.r#type.as_str(), entry.key.as_str()), entry); + } + let mut change = IndexChange::default(); + let mut groups: Vec<(IndexSlot, BTreeSet)> = Vec::new(); + let mut at: BTreeMap = BTreeMap::new(); + for entry in last.values() { + if !self.projectors.contains_key(&entry.r#type) { + continue; + } + if replacing && let Some(old) = self.entries.get(&entry_id(entry)) { + change.released.extend(self.projected_slots(old)?); + } + for (slot, key) in self.projected_slots(entry)? { + let index = *at.entry(slot.clone()).or_insert_with(|| { + groups.push((slot, BTreeSet::new())); + groups.len() - 1 + }); + groups[index].1.insert(key); + } + } + let mut least: Option<[&str; 4]> = None; + for (slot, holders) in &mut groups { + if let Some(held_key) = held.get(&*slot) + && !last.contains_key(&(slot.0.as_str(), held_key.as_str())) + { + holders.insert(held_key.clone()); + } + } + let mut ordered: Vec<(&IndexSlot, Vec<&String>)> = Vec::new(); + for (slot, holders) in &groups { + let mut keys: Vec<&String> = holders.iter().collect(); + keys.sort_by(|left, right| utf16_cmp(left, right)); + ordered.push((slot, keys)); + } + for (slot, keys) in &ordered { + if let [first, second, ..] = keys.as_slice() { + let schema = self.definitions.get(&slot.0).copied().unwrap_or(&slot.0); + let violation = [slot.1.as_str(), first.as_str(), second.as_str(), schema]; + if least.is_none_or(|best| violation_cmp(&violation, &best).is_lt()) { + least = Some(violation); + } + } + } + if let Some([alias, first, second, schema]) = least { + return Err(anyhow!( + "Records {first} and {second} ({schema}) both hold unique index '{alias}'; an index value names one record." + )); + } + for (slot, mut keys) in ordered.into_iter().map(|(slot, keys)| (slot.clone(), keys)) { + let key = keys.remove(0).clone(); + change.taken.push((slot, key)); + } + Ok(change) + } + fn refuse_second_global( &self, entries: &BTreeMap<(String, String), CultCacheEnvelope>, @@ -6136,6 +6322,27 @@ mod tests { Ok(()) } + #[test] + fn pulling_a_synthetic_pre_v1_envelope_array_leaves_the_file_bytes_identical() -> Result<()> { + // SYNTHETIC pre-v1 vector: no real file of this shape is known; the Rust envelope payload is binary. + let temp = tempfile::tempdir()?; + let path = temp.path().join("legacy.cc"); + let legacy = rmp_serde::to_vec_named(&vec![CultCacheEnvelope { + key: "note:hello".to_string(), + r#type: "note".to_string(), + payload: vec![0x90], + stored_at: "2026-10-20T00:00:00Z".to_string(), + schema_id: None, + }])?; + fs::write(&path, &legacy)?; + + let envelopes = SingleFileMessagePackBackingStore::new(&path).pull_all()?; + + assert_eq!(envelopes.iter().map(|e| e.key.as_str()).collect::>(), ["note:hello"]); + assert_eq!(fs::read(&path)?, legacy); + Ok(()) + } + #[test] fn a_newer_store_is_named_once_its_header_string_is_complete() -> Result<()> { let vector = fs::read( diff --git a/packages/cultcache-rs/tests/lookups.rs b/packages/cultcache-rs/tests/lookups.rs index ca5b459d..29aa3489 100644 --- a/packages/cultcache-rs/tests/lookups.rs +++ b/packages/cultcache-rs/tests/lookups.rs @@ -11,7 +11,7 @@ use cultcache_rs::{ struct Item { #[cultcache(key = 0, name)] name: String, - #[cultcache(key = 1, index)] + #[cultcache(key = 1)] category: String, #[cultcache(key = 2, index = "sku")] code: Option, @@ -30,7 +30,7 @@ struct Gadget { #[cultcache(key = 0, name)] name: String, #[cultcache(key = 1, index)] - category: String, + serial: String, } /// Sorts after `Item` and has its exact shape: a lookup that ran past the end of its own type would answer from it. @@ -39,7 +39,7 @@ struct Gadget { struct Zed { #[cultcache(key = 0, name)] name: String, - #[cultcache(key = 1, index)] + #[cultcache(key = 1)] category: String, #[cultcache(key = 2, index = "sku")] code: Option, @@ -48,7 +48,7 @@ struct Zed { } #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] -#[cultcache(type = "settings", global)] +#[cultcache(type = "settings", schema = "tests.settings", global)] struct Settings { #[cultcache(key = 0)] value: String, @@ -71,7 +71,7 @@ struct Numbered { #[cultcache(key = 0, name)] id: u32, #[cultcache(key = 1, index)] - owner: u64, + badge: u64, } fn cache() -> CultCache { @@ -110,7 +110,6 @@ fn a_record_is_found_by_name_and_by_index_after_a_reopen() { let reopened = reopened(&path); let by_name = reopened.get_by_name::("Potion").unwrap().unwrap(); assert_eq!(by_name.value, 50); - assert_eq!(reopened.get_by_index::("category", "Consumable").unwrap(), Some(by_name.clone())); assert_eq!(reopened.get_by_index::("sku", "P-1").unwrap(), Some(by_name)); } @@ -121,31 +120,32 @@ fn a_missing_name_or_value_is_none_and_an_unregistered_type_is_refused() { assert_eq!(cache.get_by_name::("B").unwrap(), None); assert_eq!(cache.get_by_name::("a").unwrap(), None, "names compare ordinally"); - assert_eq!(cache.get_by_index::("category", "other").unwrap(), None); - assert_eq!(cache.get_by_index::("nosuchalias", "cat").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", "other").unwrap(), None); + assert_eq!(cache.get_by_index::("nosuchalias", "a-1").unwrap(), None); + assert_eq!(cache.get_by_index::("category", "cat").unwrap(), None, "category is a grouping, not an index"); assert_eq!(cache.get_by_index::("code", "a-1").unwrap(), None, "the alias is sku, not the member name"); let bare = CultCache::new(); assert!(bare.get_by_name::("A").is_err()); - assert!(bare.get_by_index::("category", "cat").is_err()); + assert!(bare.get_by_index::("sku", "a-1").is_err()); assert!(bare.get_global::().is_err()); } #[test] fn an_index_without_an_alias_is_named_by_its_member() { let mut cache = cache(); - cache.put("item:a", &item("A", "cat", None)).unwrap(); - assert!(cache.get_by_index::("category", "cat").unwrap().is_some()); + cache.put("numbered:1", &Numbered { id: 1, badge: 7 }).unwrap(); + assert!(cache.get_by_index::("badge", "7").unwrap().is_some()); } // A blank name is no name; a blank index value is not indexed; a nil member holds no value. #[test] fn blank_and_nil_values_are_not_found() { let mut cache = cache(); - cache.put("item:blank", &item("", " ", None)).unwrap(); + cache.put("item:blank", &item("", " ", Some(" "))).unwrap(); assert_eq!(cache.get_by_name::("").unwrap(), None); - assert_eq!(cache.get_by_index::("category", " ").unwrap(), None); - assert_eq!(cache.get_by_index::("category", "").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", " ").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", "").unwrap(), None); assert_eq!(cache.get_by_index::("sku", "").unwrap(), None); // Only emptiness voids a name: whitespace is a name, as in C# (Length > 0). @@ -156,68 +156,41 @@ fn blank_and_nil_values_are_not_found() { #[test] fn lookups_never_cross_types() { let mut cache = cache(); - cache.put("item:a", &item("same", "same", None)).unwrap(); - cache.put("gadget:a", &Gadget { name: "same".into(), category: "same".into() }).unwrap(); + cache.put("item:a", &item("same", "same", Some("same"))).unwrap(); + cache.put("gadget:a", &Gadget { name: "same".into(), serial: "same".into() }).unwrap(); cache.put("zed:a", &Zed { name: "same".into(), category: "same".into(), code: None, value: 0 }).unwrap(); assert_eq!(cache.get_by_name::("same").unwrap().unwrap().name, "same"); cache.delete::("item:a").unwrap(); assert_eq!(cache.get_by_name::("same").unwrap(), None); - assert_eq!(cache.get_by_index::("category", "same").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", "same").unwrap(), None); assert!(cache.get_by_name::("same").unwrap().is_some()); - assert!(cache.get_by_index::("category", "same").unwrap().is_some()); + assert!(cache.get_by_index::("serial", "same").unwrap().is_some()); } #[test] fn a_name_and_index_follow_a_rewritten_record_and_leave_a_removed_one() { let mut cache = cache(); - cache.put("item:a", &item("old", "c1", None)).unwrap(); - cache.put("item:a", &item("new", "c2", None)).unwrap(); + cache.put("item:a", &item("old", "cat", Some("c1"))).unwrap(); + cache.put("item:a", &item("new", "cat", Some("c2"))).unwrap(); assert_eq!(cache.get_by_name::("old").unwrap(), None, "a stale name is not found"); - assert_eq!(cache.get_by_index::("category", "c1").unwrap(), None, "a stale value is not found"); + assert_eq!(cache.get_by_index::("sku", "c1").unwrap(), None, "a stale value is not found"); assert!(cache.get_by_name::("new").unwrap().is_some()); - assert!(cache.get_by_index::("category", "c2").unwrap().is_some()); + assert!(cache.get_by_index::("sku", "c2").unwrap().is_some()); cache.delete::("item:a").unwrap(); assert_eq!(cache.get_by_name::("new").unwrap(), None); - assert_eq!(cache.get_by_index::("category", "c2").unwrap(), None); + assert_eq!(cache.get_by_index::("sku", "c2").unwrap(), None); } #[test] fn non_string_members_are_found_by_their_text() { let mut cache = cache(); - let owner = 161_u64; - cache.put("numbered:1", &Numbered { id: 42, owner }).unwrap(); + let badge = 161_u64; + cache.put("numbered:1", &Numbered { id: 42, badge }).unwrap(); assert!(cache.get_by_name::("42").unwrap().is_some()); - assert!(cache.get_by_index::("owner", "161").unwrap().is_some()); -} - -// VariantTests.RemovingTheHolderADuplicateValueFindsLeavesTheOtherFindable: records sharing a value find the one with -// the smallest key, whatever the write order, live and after a reopen; removing the holder finds the other. -#[test] -fn records_sharing_a_value_find_the_smallest_key_and_removing_it_finds_the_next() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("shared.cc"); - { - let mut cache = stored_cache(&path); - cache.put("item:p2", &item("p2", "x", None)).unwrap(); - cache.put("item:p1", &item("p1", "x", None)).unwrap(); - let found = |cache: &CultCache| cache.get_by_index::("category", "x").unwrap().map(|found| found.name); - assert_eq!(found(&cache).as_deref(), Some("p1"), "the smallest key, though written second"); - assert_eq!(reopened(&path).get_by_index::("category", "x").unwrap().unwrap().name, "p1"); - - cache.delete::("item:p1").unwrap(); - assert_eq!(found(&cache).as_deref(), Some("p2"), "removing the holder finds the other one"); - - cache.put("item:p1", &item("p1", "x", None)).unwrap(); - assert_eq!(found(&cache).as_deref(), Some("p1")); - cache.delete::("item:p2").unwrap(); - assert_eq!(found(&cache).as_deref(), Some("p1"), "removing the other changes nothing"); - cache.delete::("item:p1").unwrap(); - assert_eq!(found(&cache), None); - } - assert_eq!(reopened(&path).get_by_index::("category", "x").unwrap(), None); + assert!(cache.get_by_index::("badge", "161").unwrap().is_some()); } #[test] @@ -244,6 +217,71 @@ fn a_global_is_none_until_written_and_survives_a_reopen() { assert_eq!(reopened(&path).get_global::().unwrap(), Some(settings("durable authority"))); } +/// `Settings` as an ordinary type: the same type id and schema, no global rule. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "settings", schema = "tests.settings")] +struct LooseSettings { + #[cultcache(key = 0)] + value: String, +} + +// Declaring a held type global judges the records it holds: two are already a second, so the registration is +// refused and the type stays ordinary. +#[test] +fn registering_a_type_global_over_two_held_records_is_refused_and_changes_nothing() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &LooseSettings { value: "one".into() }).unwrap(); + cache.put("b", &LooseSettings { value: "two".into() }).unwrap(); + + let refusal = cache.register_entry_type::().unwrap_err(); + assert!(refusal.downcast_ref::().is_some(), "{refusal:#}"); + + cache.put("c", &LooseSettings { value: "three".into() }).unwrap(); + assert_eq!(cache.get_all_with_keys::().unwrap().len(), 3, "still ordinary"); +} + +// One held record may become the global; a second is then refused as a second global. +#[test] +fn registering_a_type_global_over_one_held_record_is_admitted() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &LooseSettings { value: "one".into() }).unwrap(); + cache.register_entry_type::().unwrap(); + let refusal = cache.put("b", &Settings { value: "two".into() }).unwrap_err(); + assert!(refusal.downcast_ref::().is_some(), "{refusal:#}"); +} + +/// A global with an indexed member, and the same type and schema declaring that member as an integer. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "indexed_global", schema = "tests.indexed_global", global)] +struct IndexedGlobal { + #[cultcache(key = 0, index = "V")] + v: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "indexed_global", schema = "tests.indexed_global", global)] +struct RetypedGlobal { + #[cultcache(key = 0, index = "V")] + v: i64, +} + +// A refused re-registration over an already-global type leaves it global: the rule keeps refusing a second record. +#[test] +fn a_refused_reregistration_leaves_a_global_type_global() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &IndexedGlobal { v: "x".into() }).unwrap(); + + assert!(cache.register_entry_type::().is_err(), "the held text cannot be decoded as an integer"); + + let refusal = cache.put("b", &IndexedGlobal { v: "y".into() }).unwrap_err(); + assert!(refusal.downcast_ref::().is_some(), "{refusal:#}"); + assert_eq!(cache.get_global::().unwrap(), Some(IndexedGlobal { v: "x".into() })); + assert_eq!(cache.get_by_index::("V", "x").unwrap(), cache.get::("a").unwrap()); +} + #[test] fn a_type_that_is_not_a_global_has_no_global() { let mut cache = cache(); diff --git a/packages/cultcache-rs/tests/unique_index.rs b/packages/cultcache-rs/tests/unique_index.rs new file mode 100644 index 00000000..1132b2b5 --- /dev/null +++ b/packages/cultcache-rs/tests/unique_index.rs @@ -0,0 +1,775 @@ +//! `#[cultcache(index)]` is unique (`variants:ruling:unique-indexes`; C# `RefuseIndexSharing`): within one type and one +//! alias no two records hold one non-blank value, judged on the result of each admission, on every write path and every +//! load. The refusal text is informational; the contract is the verdict, so most pins assert the verdict and the +//! shared vectors in `tests/vectors/unique-index` pin the text and the canonical index text. + +use std::path::{Path, PathBuf}; + +use cultcache_rs::{ + CultCache, CultCacheEnvelope, CultCacheSoaTable, DatabaseEntry, SingleFileMessagePackBackingStore, +}; +use serde::Deserialize; + +/// The vectors' `tests.unique_item`: Sku is unique, Category a grouping. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "unique_item", schema = "tests.unique_item")] +struct Unique { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + category: String, +} + +/// The same type with no index: a role, which writes the duplicates `Unique` refuses. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "unique_item", schema = "tests.unique_item")] +struct Twin { + #[cultcache(key = 0)] + sku: String, + #[cultcache(key = 1)] + category: String, +} + +/// Two unique indexes declared out of alphabetical order, so no ordering is an accident of declaration. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "multi", schema = "Multi")] +struct Multi { + #[cultcache(key = 0, index = "B")] + b: Option, + #[cultcache(key = 1, index = "A")] + a: Option, +} + +/// Another type sharing the alias `Sku`: types never conflict. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "other", schema = "Other")] +struct Other { + #[cultcache(key = 0, index = "Sku")] + sku: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "plain", schema = "Plain")] +struct Plain { + #[cultcache(key = 0)] + value: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "vs", schema = "VS")] +struct VString { + #[cultcache(key = 0, index = "V")] + v: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "vi", schema = "VI")] +struct VInt { + #[cultcache(key = 0, index = "V")] + v: i64, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "vu", schema = "VU")] +struct VUint { + #[cultcache(key = 0, index = "V")] + v: u64, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "vb", schema = "VB")] +struct VBool { + #[cultcache(key = 0, index = "V")] + v: bool, +} + +/// The refusals vector's other schemas: `Sku` is unique in each; `Alt` is a second unique index in `uq_pair` (alias +/// `Alt`) and `uq_alias` (alias `alt`, sorting after `Sku` only under an ordinal compare); alpha and beta are plain. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_pair", schema = "tests.uq_pair")] +struct UqPair { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1, index = "Alt")] + alt: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_alias", schema = "tests.uq_alias")] +struct UqAlias { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1, index = "alt")] + alt: String, +} +/// Aliases U+FF21 and U+10400: UTF-16 puts the astral alias first (D801 DC00 < FF21), code-point order the other way. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_wide", schema = "tests.uq_wide")] +struct UqWide { + #[cultcache(key = 0, index = "\u{FF21}")] + sku: String, + #[cultcache(key = 1, index = "\u{10400}")] + alt: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_alpha", schema = "tests.uq_alpha")] +struct UqAlpha { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + second: String, +} +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "uq_beta", schema = "tests.uq_beta")] +struct UqBeta { + #[cultcache(key = 0, index = "Sku")] + sku: String, + #[cultcache(key = 1)] + second: String, +} + +fn unique(sku: &str) -> Unique { + Unique { sku: sku.into(), category: "c".into() } +} + +fn multi(a: Option<&str>, b: Option<&str>) -> Multi { + Multi { a: a.map(Into::into), b: b.map(Into::into) } +} + +fn refusal(k1: &str, k2: &str, schema: &str, alias: &str) -> String { + format!("Records {k1} and {k2} ({schema}) both hold unique index '{alias}'; an index value names one record.") +} + +fn cache_at(path: &Path) -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache +} + +fn memory_cache() -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache +} + +fn twin_cache_at(path: &Path) -> CultCache { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache +} + +fn error_text(result: anyhow::Result) -> String { + format!("{:#}", result.unwrap_err()) +} + +fn skus(cache: &CultCache) -> Vec<(String, String)> { + cache.get_all_with_keys::().unwrap().into_iter().map(|(key, value)| (key, value.sku)).collect() +} + +#[test] +fn the_derive_emits_the_unique_aliases() { + fn aliases() -> Vec<&'static str> { + T::UNIQUE_INDEXES.iter().map(|(alias, _)| *alias).collect() + } + assert_eq!(aliases::(), ["Sku"]); + assert_eq!(aliases::(), ["B", "A"]); + assert!(aliases::().is_empty()); +} + +#[test] +fn a_put_of_a_held_value_at_another_key_is_refused_and_changes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("u.cc"); + let mut cache = cache_at(&path); + cache.put("item:b", &unique("S-1")).unwrap(); + let bytes = std::fs::read(&path).unwrap(); + + let text = error_text(cache.put("item:a", &unique("S-1"))); + assert_eq!(text, refusal("item:a", "item:b", "tests.unique_item", "Sku")); + assert!(!text.contains("S-1"), "the refusal never names the value"); + assert_eq!(std::fs::read(&path).unwrap(), bytes, "the store is untouched"); + assert_eq!(cache.get::("item:a").unwrap(), None); + assert_eq!(cache.get_by_index::("Sku", "S-1").unwrap().unwrap().sku, "S-1"); + assert_eq!(cache.get_by_index::("Sku", "S-1").unwrap(), cache.get::("item:b").unwrap()); +} + +// The old bug: write b then a sharing a value used to answer whichever key sorted first. +#[test] +fn the_first_holder_keeps_the_value_whatever_the_key_order() { + let mut cache = memory_cache(); + cache.put("item:z", &unique("S")).unwrap(); + assert!(cache.put("item:a", &unique("S")).is_err()); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap().unwrap(), unique("S")); + assert_eq!(skus(&cache), vec![("item:z".to_string(), "S".to_string())]); +} + +#[test] +fn a_record_rewriting_its_own_key_keeps_its_value_and_a_change_releases_the_old_one() { + let mut cache = memory_cache(); + cache.put("k", &unique("S")).unwrap(); + cache.put("k", &Unique { sku: "S".into(), category: "other".into() }).unwrap(); + assert!(cache.get_by_index::("Sku", "S").unwrap().is_some()); + + cache.put("k", &unique("T")).unwrap(); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), None, "a stale value is not found"); + cache.put("j", &unique("S")).unwrap(); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap().unwrap().sku, "S"); + assert_eq!(cache.get_by_index::("Sku", "T").unwrap().unwrap().sku, "T"); + // Updating through `update` is the same rule. + assert!(cache.update::("k", |_| unique("S")).is_err()); + assert_eq!(cache.get::("k").unwrap().unwrap().sku, "T"); +} + +#[test] +fn delete_releases_a_value() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("d.cc"); + let mut cache = cache_at(&path); + cache.put("item:a", &unique("S")).unwrap(); + assert!(cache.put("item:b", &unique("S")).is_err()); + assert!(cache.delete::("item:a").unwrap()); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), None); + cache.put("item:b", &unique("S")).unwrap(); + assert!(cache.get_by_index::("Sku", "S").unwrap().is_some()); + assert!(!cache.delete::("item:a").unwrap(), "deleting an absent key releases nothing"); + assert!(cache.get_by_index::("Sku", "S").unwrap().is_some()); + assert!(cache_at(&path).get_by_index::("Sku", "S").unwrap().is_some()); +} + +#[test] +fn types_sharing_an_alias_and_a_value_do_not_conflict() { + let mut cache = memory_cache(); + cache.put("a", &unique("S")).unwrap(); + cache.put("a", &Other { sku: "S".into() }).unwrap(); + cache.put("b", &Other { sku: "T".into() }).unwrap(); + assert!(cache.put("b", &Other { sku: "S".into() }).is_err()); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap().unwrap(), unique("S")); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap().unwrap(), Other { sku: "S".into() }); +} + +#[test] +fn an_alias_the_type_does_not_declare_finds_nothing() { + let mut cache = memory_cache(); + cache.put("a", &unique("S")).unwrap(); + assert_eq!(cache.get_by_index::("sku", "S").unwrap(), None); + assert_eq!(cache.get_by_index::("Category", "c").unwrap(), None); +} + +#[test] +fn blank_and_none_values_never_conflict() { + let mut cache = memory_cache(); + for (key, sku) in [("a", " "), ("b", ""), ("c", "\u{85}"), ("d", "\t\n\u{a0}\u{3000}")] { + cache.put(key, &unique(sku)).unwrap(); + } + cache.put("m1", &multi(None, None)).unwrap(); + cache.put("m2", &multi(None, Some(" "))).unwrap(); + assert_eq!(cache.get_by_index::("Sku", " ").unwrap(), None); + // U+FEFF, U+001F, U+200B, U+180E are values, never trimmed. + cache.put("e", &unique("\u{feff}")).unwrap(); + assert!(cache.put("f", &unique("\u{feff}")).is_err()); + cache.put("g", &unique(" S ")).unwrap(); + cache.put("h", &unique("S")).unwrap(); + assert_eq!(cache.get_by_index::("Sku", " S ").unwrap().unwrap().sku, " S "); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap().unwrap().sku, "S"); +} + +// C# `string.IsNullOrWhiteSpace` over every code unit: exactly these 25 are blank (probed at 7760219b, dotnet 10). +#[test] +fn the_blank_set_is_exactly_the_csharp_white_space() { + let blank: Vec = (0x9..=0xD) + .chain([0x20, 0x85, 0xA0, 0x1680]) + .chain(0x2000..=0x200A) + .chain([0x2028, 0x2029, 0x202F, 0x205F, 0x3000]) + .collect(); + assert_eq!(blank.len(), 25); + let mut cache = memory_cache(); + let mut found = Vec::new(); + for unit in (0..=0xFFFF_u32).filter(|unit| char::from_u32(*unit).is_some()) { + let text = char::from_u32(unit).unwrap().to_string(); + cache.put("a", &unique(&text)).unwrap(); + let second = cache.put("b", &unique(&text)); + if second.is_ok() { + found.push(unit); + cache.delete::("b").unwrap(); + assert_eq!(cache.get_by_index::("Sku", &text).unwrap(), None, "U+{unit:04X} blank is never found"); + } else { + assert_eq!(cache.get_by_index::("Sku", &text).unwrap().unwrap().sku, text); + } + } + assert_eq!(found, blank); +} + +fn staged(cache: &CultCache, rows: &[(&str, T)]) -> Vec { + rows.iter().map(|(key, value)| cache.prepare_entry(*key, value).unwrap().0).collect() +} + +// Two unique indexes, collision on the second-declared, through every write path. +#[test] +fn every_write_path_refuses_and_changes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("paths.cc"); + let mut cache = cache_at(&path); + cache.put("m:held", &multi(Some("a1"), Some("b1"))).unwrap(); + let bytes = std::fs::read(&path).unwrap(); + let before = cache.snapshot(); + + let clash_b = multi(Some("a2"), Some("b1")); + let clash_a = multi(Some("a1"), Some("b2")); + for (clash, alias) in [(&clash_b, "B"), (&clash_a, "A")] { + let pick = |text: &str| { + let (first, second) = if "m:new" < "m:held" { ("m:new", "m:held") } else { ("m:held", "m:new") }; + assert_eq!(text, refusal(first, second, "Multi", alias)); + }; + pick(&error_text(cache.put("m:new", clash))); + pick(&error_text(cache.update::("m:new", |_| clash.clone()))); + let envelope = staged(&cache, &[("m:new", clash.clone())]).remove(0); + pick(&error_text(cache.put_envelope::(envelope.clone()))); + pick(&error_text(cache.put_raw_envelope(envelope.clone()))); + pick(&error_text(cache.put_prepared_batch(vec![envelope]))); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert_eq!(cache.snapshot(), before); + assert_eq!(cache.get_by_index::("B", "b2").unwrap(), None); + } + // A clean row through every path lands and is found by both aliases. + let fine = multi(Some("a9"), Some("b9")); + cache.put("m:1", &fine).unwrap(); + let envelope = staged(&cache, &[("m:2", multi(Some("a8"), Some("b8")))]).remove(0); + cache.put_envelope::(envelope).unwrap(); + let envelope = staged(&cache, &[("m:3", multi(Some("a7"), Some("b7")))]).remove(0); + cache.put_raw_envelope(envelope).unwrap(); + cache.put_prepared_batch(staged(&cache, &[("m:4", multi(Some("a6"), Some("b6")))])).unwrap(); + for (alias, value, key) in [("A", "a9", 1), ("B", "b8", 2), ("A", "a7", 3), ("B", "b6", 4)] { + let found = cache.get_by_index::(alias, value).unwrap().unwrap(); + assert_eq!(Some(found), cache.get::(&format!("m:{key}")).unwrap()); + } + // Held values survive a reopen of the store: the load installs the same holders. + let mut reopened = cache_at(&path); + assert!(reopened.put("m:x", &multi(None, Some("b9"))).is_err()); + assert!(reopened.put("m:x", &multi(Some("a6"), None)).is_err()); + assert!(reopened.put("m:x", &multi(Some("zz"), Some("yy"))).is_ok()); +} + +#[test] +fn a_batch_is_judged_whole() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("batch.cc"); + let mut cache = cache_at(&path); + cache.put("a", &unique("X")).unwrap(); + cache.put("b", &unique("Y")).unwrap(); + + // A swap inside one batch lands. + cache.put_prepared_batch(staged(&cache, &[("a", unique("Y")), ("b", unique("X"))])).unwrap(); + assert_eq!(cache.get_by_index::("Sku", "Y").unwrap(), cache.get::("a").unwrap()); + assert_eq!(cache.get_by_index::("Sku", "X").unwrap(), cache.get::("b").unwrap()); + assert_eq!(cache_at(&path).get_by_index::("Sku", "X").unwrap().unwrap().sku, "X"); + + // A batch taking a value a record being replaced just released lands; one holding a value twice is refused whole. + cache.put_prepared_batch(staged(&cache, &[("a", unique("Z")), ("c", unique("Y"))])).unwrap(); + let bytes = std::fs::read(&path).unwrap(); + let before = cache.snapshot(); + let text = error_text(cache.put_prepared_batch(staged(&cache, &[("p", unique("N")), ("q", unique("N")), ("r", unique("W"))]))); + assert_eq!(text, refusal("p", "q", "tests.unique_item", "Sku")); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert_eq!(cache.snapshot(), before); + + // A key staged twice lands as its last envelope. + cache.put_prepared_batch(staged(&cache, &[("d", unique("D1")), ("d", unique("D2")), ("e", unique("D1"))])).unwrap(); + assert_eq!(cache.get_by_index::("Sku", "D2").unwrap(), cache.get::("d").unwrap()); + assert_eq!(cache.get_by_index::("Sku", "D1").unwrap(), cache.get::("e").unwrap()); +} + +#[test] +fn a_load_of_a_duplicate_image_is_refused_and_keeps_the_prior_view() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("load.cc"); + let mut cache = cache_at(&path); + cache.put("item:keep", &unique("K")).unwrap(); + + let mut twin = twin_cache_at(&path); + twin.put("item:b", &Twin { sku: "S".into(), category: "c".into() }).unwrap(); + twin.put("item:a", &Twin { sku: "S".into(), category: "c".into() }).unwrap(); + let bytes = std::fs::read(&path).unwrap(); + + let before = cache.snapshot(); + assert_eq!(error_text(cache.pull_all_backing_stores()), refusal("item:a", "item:b", "tests.unique_item", "Sku")); + assert_eq!(cache.snapshot(), before, "a refused pull admits nothing"); + assert_eq!(cache.get_by_index::("Sku", "K").unwrap().unwrap().sku, "K"); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + + // load_soa of the same image through a cache with no store. + let table: CultCacheSoaTable = twin.snapshot_soa(); + let mut soa = memory_cache(); + soa.put("item:keep", &unique("K")).unwrap(); + let before = soa.snapshot(); + assert_eq!(error_text(soa.load_soa(table)), refusal("item:a", "item:b", "tests.unique_item", "Sku")); + assert_eq!(soa.snapshot(), before); + assert!(soa.get_by_index::("Sku", "K").unwrap().is_some()); + + // A clean image loads and replaces the holders with the image's. + let mut clean = twin_cache_at(&dir.path().join("clean.cc")); + clean.put("item:x", &Twin { sku: "X".into(), category: "c".into() }).unwrap(); + soa.load_soa(clean.snapshot_soa()).unwrap(); + assert_eq!(soa.get_by_index::("Sku", "K").unwrap(), None); + assert!(soa.get_by_index::("Sku", "X").unwrap().is_some()); + soa.put("item:y", &unique("K")).unwrap(); +} + +#[test] +fn a_multi_index_store_with_a_duplicate_on_the_second_alias_refuses_to_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("multi.cc"); + { + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "multi", schema = "Multi")] + struct Role { + #[cultcache(key = 0)] + b: Option, + #[cultcache(key = 1)] + a: Option, + } + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores().unwrap(); + cache.put("m:1", &Role { b: Some("b".into()), a: Some("a1".into()) }).unwrap(); + cache.put("m:2", &Role { b: Some("b".into()), a: Some("a2".into()) }).unwrap(); + } + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + assert_eq!(error_text(cache.pull_all_backing_stores()), refusal("m:1", "m:2", "Multi", "B")); +} + +#[test] +fn a_bool_index_is_found_by_the_csharp_text() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("t", &VBool { v: true }).unwrap(); + assert!(cache.put("u", &VBool { v: true }).is_err()); + cache.put("f", &VBool { v: false }).unwrap(); + assert_eq!(cache.get_by_index::("V", "True").unwrap().unwrap().v, true); + assert_eq!(cache.get_by_index::("V", "False").unwrap().unwrap().v, false); + assert_eq!(cache.get_by_index::("V", "true").unwrap(), None); + assert_eq!(cache.get_by_index::("V", "false").unwrap(), None); +} + +// ---- shared vectors, written by the C# reference ---- + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum Cell { + Nil, + Bool(bool), + Int(i64), + Uint(u64), + Double(f64), + Text(String), +} + +fn vectors() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/unique-index") +} + +type ValueRow = (String, String, Cell, Option); + +#[test] +fn vectors_values_give_the_csharp_index_text() { + let bytes = std::fs::read(vectors().join("values.msgpack")).unwrap(); + let rows: Vec = rmp_serde::from_slice(&bytes).unwrap(); + assert!(!rows.is_empty()); + let mut checked = 0; + for (label, kind, value, text) in rows { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + cache.register_entry_type::().unwrap(); + // `double` rows are skipped: CultLookupValue has no float impl, so Rust cannot index a double member. + match (kind.as_str(), value) { + ("string", Cell::Text(held)) => { + cache.put("one", &VString { v: held.clone() }).unwrap(); + match text { + Some(text) => { + assert_eq!(cache.get_by_index::("V", &text).unwrap().unwrap().v, held, "{label}"); + assert!(cache.put("two", &VString { v: held }).is_err(), "{label}"); + } + None => { + cache.put("two", &VString { v: held.clone() }).unwrap(); + assert_eq!(cache.get_by_index::("V", &held).unwrap(), None, "{label}"); + } + } + } + ("int64", Cell::Int(held)) => { + cache.put("one", &VInt { v: held }).unwrap(); + assert_eq!(cache.get_by_index::("V", &text.unwrap()).unwrap().unwrap().v, held, "{label}"); + assert!(cache.put("two", &VInt { v: held }).is_err(), "{label}"); + } + ("uint64", Cell::Uint(held)) => { + cache.put("one", &VUint { v: held }).unwrap(); + assert_eq!(cache.get_by_index::("V", &text.unwrap()).unwrap().unwrap().v, held, "{label}"); + assert!(cache.put("two", &VUint { v: held }).is_err(), "{label}"); + } + ("uint64", Cell::Int(held)) => { + cache.put("one", &VUint { v: held as u64 }).unwrap(); + assert_eq!(cache.get_by_index::("V", &text.unwrap()).unwrap().unwrap().v, held as u64, "{label}"); + } + ("int64", Cell::Uint(held)) => { + cache.put("one", &VInt { v: held as i64 }).unwrap(); + assert_eq!(cache.get_by_index::("V", &text.unwrap()).unwrap().unwrap().v, held as i64, "{label}"); + } + ("bool", Cell::Bool(held)) => { + cache.put("one", &VBool { v: held }).unwrap(); + assert_eq!(cache.get_by_index::("V", &text.unwrap()).unwrap().unwrap().v, held, "{label}"); + } + ("double", Cell::Double(_) | Cell::Int(_) | Cell::Uint(_) | Cell::Nil) => continue, + (kind, value) => panic!("{label}: unexpected vector row {kind} {value:?}"), + } + checked += 1; + } + assert!(checked > 10, "the vectors were read, not skipped: {checked}"); +} + +type OutcomeRow = (String, Option, String, Option, Option); + +#[test] +fn vectors_stores_hold_their_verdicts() { + let bytes = std::fs::read(vectors().join("outcomes.msgpack")).unwrap(); + let rows: Vec = rmp_serde::from_slice(&bytes).unwrap(); + assert!(rows.len() >= 3); + let dir = tempfile::tempdir().unwrap(); + for (file, alias, verdict, k1, k2) in rows { + let path = dir.path().join(&file); + std::fs::copy(vectors().join(&file), &path).unwrap(); + let before = std::fs::read(&path).unwrap(); + let attach = |cache: &mut CultCache| { + cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + cache.pull_all_backing_stores() + }; + let mut cache = CultCache::new(); + let opened = match alias.as_deref() { + Some("Sku") => { + cache.register_entry_type::().unwrap(); + attach(&mut cache) + } + None => { + cache.register_entry_type::().unwrap(); + attach(&mut cache) + } + Some(other) => panic!("unexpected vector alias {other}"), + }; + match verdict.as_str() { + "open" => opened.unwrap_or_else(|error| panic!("{file} / {alias:?} should open: {error:#}")), + "refused" => assert_eq!( + error_text(opened), + refusal(&k1.unwrap(), &k2.unwrap(), "tests.unique_item", "Sku"), + "{file}" + ), + other => panic!("unexpected verdict {other}"), + } + assert_eq!(std::fs::read(&path).unwrap(), before, "{file} is byte-identical"); + } +} + +/// One op of a refusals row: `["open", file-or-nil]`, `["pull", file]` or `["commit", [[key, schema, sku, second], ...]]`. +#[derive(Deserialize)] +#[serde(untagged)] +enum Arg { + Records(Vec<(String, String, String, String)>), + File(Option), +} + +type RefusalRow = (String, Vec<(String, Arg)>, Option<(String, String, String, String)>); + +/// Rows `[label, ops, refusal]`: the pair a refusal names is a function of the state the admission would leave +/// (`tests/vectors/unique-index/README.md`, RULE), whatever the staging, file or pull order, key case or alias case. +#[test] +fn vectors_refusals_hold_the_rule() { + let bytes = std::fs::read(vectors().join("refusals.msgpack")).unwrap(); + let rows: Vec = rmp_serde::from_slice(&bytes).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let mut walked = 0; + for (index, (label, ops, refused)) in rows.into_iter().enumerate() { + let path = dir.path().join(format!("row-{index}.cc")); + let mut cache: Option = None; + let mut failure = None; + for (at, (kind, arg)) in ops.iter().enumerate() { + let last = at + 1 == ops.len(); + let mut before = path.exists().then(|| std::fs::read(&path).unwrap()); + let result = match (kind.as_str(), arg) { + ("open", Arg::File(file)) => { + if let Some(file) = file { + std::fs::copy(vectors().join(file), &path).unwrap(); + } + let mut opened = CultCache::new(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.register_entry_type::().unwrap(); + opened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap(); + let pulled = opened.pull_all_backing_stores(); + cache = Some(opened); + pulled + } + ("pull", Arg::File(Some(file))) => { + std::fs::copy(vectors().join(file), &path).unwrap(); + before = Some(std::fs::read(&path).unwrap()); + cache.as_mut().unwrap().pull_all_backing_stores() + } + ("commit", Arg::Records(records)) => { + let cache = cache.as_mut().unwrap(); + let staged: Vec<_> = records + .iter() + .map(|(key, schema, sku, second)| { + let (sku, second) = (sku.clone(), second.clone()); + match schema.as_str() { + "tests.unique_item" => cache.prepare_entry(key, &Unique { sku, category: second }).map(|(envelope, _)| envelope), + "tests.uq_pair" => cache.prepare_entry(key, &UqPair { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_alias" => cache.prepare_entry(key, &UqAlias { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_wide" => cache.prepare_entry(key, &UqWide { sku, alt: second }).map(|(envelope, _)| envelope), + "tests.uq_alpha" => cache.prepare_entry(key, &UqAlpha { sku, second }).map(|(envelope, _)| envelope), + "tests.uq_beta" => cache.prepare_entry(key, &UqBeta { sku, second }).map(|(envelope, _)| envelope), + other => panic!("{label}: unexpected schema {other}"), + } + .unwrap() + }) + .collect(); + cache.put_prepared_batch(staged) + } + other => panic!("{label}: unexpected op {:?}", other.0), + }; + if last { + failure = Some((result, before)); + } else { + result.unwrap_or_else(|error| panic!("{label}: op {at} should succeed: {error:#}")); + } + } + let (result, before) = failure.unwrap(); + match refused { + None => result.unwrap_or_else(|error| panic!("{label}: should succeed: {error:#}")), + Some((k1, k2, schema, alias)) => { + let error = result.err().unwrap_or_else(|| panic!("{label}: should refuse")); + assert_eq!(format!("{error:#}"), refusal(&k1, &k2, &schema, &alias), "{label}"); + if let Some(before) = before { + assert_eq!(std::fs::read(&path).unwrap(), before, "{label}: the file is byte-identical"); + } + } + } + walked += 1; + } + assert_eq!(walked, 37); +} + +// A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept. +#[test] +fn a_second_pull_replaces_the_holders_with_the_stored_image() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pull.cc"); + let mut cache = cache_at(&path); + cache.put("a", &unique("S")).unwrap(); + + cache_at(&path).put("a", &unique("T")).unwrap(); + cache.pull_all_backing_stores().unwrap(); + + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), None, "the replaced value is no longer held"); + assert_eq!(cache.get_by_index::("Sku", "T").unwrap(), cache.get::("a").unwrap()); + cache.put("b", &unique("S")).unwrap(); + assert!(cache.put("c", &unique("T")).is_err()); +} + +// The projector decides what held records hold: declaring an index for a type that already has records derives +// their holders, or refuses the registration when they collide. +#[test] +fn registering_an_index_over_held_records_derives_their_holders() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &Twin { sku: "S".into(), category: "x".into() }).unwrap(); + cache.put("b", &Twin { sku: "T".into(), category: "x".into() }).unwrap(); + + cache.register_entry_type::().unwrap(); + + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), cache.get::("a").unwrap()); + assert!(cache.get_by_index::("Sku", "S").unwrap().is_some()); + assert!(cache.put("c", &unique("S")).is_err(), "a held value is refused"); + cache.put("c", &unique("U")).unwrap(); +} + +#[test] +fn registering_an_index_over_colliding_held_records_is_refused_and_changes_nothing() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &Twin { sku: "S".into(), category: "x".into() }).unwrap(); + cache.put("b", &Twin { sku: "S".into(), category: "y".into() }).unwrap(); + + let text = error_text(cache.register_entry_type::()); + assert_eq!(text, refusal("a", "b", "tests.unique_item", "Sku")); + + // Still the unindexed declaration: duplicates keep landing, nothing is looked up. + cache.put("c", &Twin { sku: "S".into(), category: "z".into() }).unwrap(); + assert_eq!(cache.get_all_with_keys::().unwrap().len(), 3); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), None); +} + +/// The same type and schema declaring Sku as an integer: the held text cannot be decoded as one. +#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] +#[cultcache(type = "unique_item", schema = "tests.unique_item")] +struct Retyped { + #[cultcache(key = 0, index = "Sku")] + sku: i64, +} + +/// A refused re-registration leaves the previous declaration in force: it keeps judging, so a later duplicate is refused. +#[test] +fn a_refused_reregistration_keeps_the_previous_index_judging() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &unique("S")).unwrap(); + cache.put("b", &unique("T")).unwrap(); + + // A different declaration over the same type (Category is the unique alias) collides on held records. + #[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)] + #[cultcache(type = "unique_item", schema = "tests.unique_item")] + struct ByCat { + #[cultcache(key = 0)] + sku: String, + #[cultcache(key = 1, index = "Cat")] + category: String, + } + assert!(cache.register_entry_type::().is_err(), "a and b both hold category c"); + + assert!(cache.put("z", &unique("S")).is_err(), "the previous index still judges"); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), cache.get::("a").unwrap()); + cache.put("z", &unique("U")).unwrap(); +} + +// A held record the new declaration cannot decode refuses the registration; the unindexed declaration stays. +#[test] +fn registering_an_index_over_an_undecodable_held_record_is_refused_and_changes_nothing() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &Twin { sku: "S".into(), category: "x".into() }).unwrap(); + + assert!(cache.register_entry_type::().is_err()); + + cache.put("b", &Twin { sku: "S".into(), category: "y".into() }).unwrap(); + assert_eq!(cache.get_all_with_keys::().unwrap().len(), 2, "no index judges the duplicate"); +} + +// Dropping the declaration releases the holders. +#[test] +fn registering_without_the_index_releases_the_holders() { + let mut cache = CultCache::new(); + cache.register_entry_type::().unwrap(); + cache.put("a", &unique("S")).unwrap(); + cache.register_entry_type::().unwrap(); + assert_eq!(cache.get_by_index::("Sku", "S").unwrap(), None, "no declaration, no holders"); + cache.put("b", &Twin { sku: "S".into(), category: "x".into() }).unwrap(); + assert!(cache.register_entry_type::().is_err(), "the duplicate now written is a held violation"); +} diff --git a/packages/cultcache-ts/CHANGELOG.md b/packages/cultcache-ts/CHANGELOG.md index 819df6c4..1d0e81a2 100644 --- a/packages/cultcache-ts/CHANGELOG.md +++ b/packages/cultcache-ts/CHANGELOG.md @@ -3,13 +3,32 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Breaking + +- A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading. +- `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank + value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical + text (booleans `True`/`False`, integers as digits, doubles as the exact expansion, blank values never indexed), so + an empty string is no longer a value. `registerIndex` over held duplicates installs nothing. A new `roles` map names + the many-row selection aliases (never indexed for lookup, same catalog `indexAlias`). The per-key migration rule is + in the README; classify every key before the CultLib your tools load is updated. The refusal verdict is the contract, its text is informational. + +### Changed + +- Opening a legacy (pre-v1) store is a pure read: payloads that were not bytes are normalized in memory and the + file is left as it was, so a codec that refuses the content cannot lose the old bytes. The next push, delete or + flush rewrites the file as v1, merging what the open read. +- `SingleFileStoreRead` no longer carries `repairedLegacyPayload`; no consumer used it. + ## [0.15.0] 0.14.1 and 0.14.2 were never tagged or published; this entry carries them with the Unreleased work, and `cultcache-ts-v0.14.0` is the previous release. ### Breaking -- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name. +- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. - A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v`. - A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. A full `pushAll` over a file with no readable stamp does not replace it: the reader refuses it. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole. - A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store. @@ -41,8 +60,6 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following - The Windows replace is a POSIX-semantics rename first (it lands while C# and Rust readers hold the store open), falling back to a write-through `MoveFileExW`, with 25 attempts 20 ms apart on errors 5, 32 and 33. Writers wait up to 30 s for the lock. -- Opening a legacy store that needs healing rewrites it under the lock, from a read taken - under the lock. ### Dependencies diff --git a/packages/cultcache-ts/README.md b/packages/cultcache-ts/README.md index b3a85345..21e76701 100644 --- a/packages/cultcache-ts/README.md +++ b/packages/cultcache-ts/README.md @@ -63,6 +63,10 @@ const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. The whole cache without the single-file store is `@gamecult/cultcache-ts/browser`, which is also what bundlers resolve for the package root (`browser` condition); bring your own `CacheBackingStore`. +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads a v3 store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-ts from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.) + ## `.cc` Inspection CultCache files may use the `.cc` extension. The bytes are still the canonical @@ -93,12 +97,16 @@ const itemDocument = defineDocumentType({ schemaVersion: "item.v1", schema: z.object({ name: z.string(), + sku: z.string(), category: z.string(), value: z.number().int(), }), name: "name", indexes: { - category: "category", + sku: "sku", // unique: a sku names one item + }, + roles: { + category: "category", // a selection alias: many items share a category }, }); @@ -122,6 +130,7 @@ await cache.pullAllBackingStores(); await cache.put(itemDocument, "item:potion", { name: "Potion", + sku: "POT-001", category: "Consumable", value: 50, }); @@ -132,7 +141,7 @@ await cache.putGlobal(settingsDocument, { }); const potion = cache.getByName(itemDocument, "Potion"); -const consumable = cache.getByIndex(itemDocument, "category", "Consumable"); +const bySku = cache.getByIndex(itemDocument, "sku", "POT-001"); const settings = cache.getRequiredGlobal(settingsDocument); ``` @@ -194,7 +203,7 @@ A document type has exactly one home store ## Name, Index, and Global Semantics -`name` and `indexes` can be declared on the document definition: +`name`, `indexes` and `roles` can be declared on the document definition: ```ts const playerDocument = defineDocumentType({ @@ -202,17 +211,70 @@ const playerDocument = defineDocumentType({ schema: playerSchema, name: "displayName", indexes: { - faction: "faction", - email: "email", + email: "email", // unique lookup + }, + roles: { + faction: "faction", // selection alias, many players share one }, }); ``` +`indexes` are unique (variants:ruling:unique-indexes; C# `[CultIndex]`, Rust `#[cultcache(index)]`). Within one type +and one alias no two records hold one non-blank value, judged on the result of every write and of every load; a +refusal reads `Records and () both hold unique index ''; an index value names one record.` +(keys in ordinal order, never the value), and writes nothing. The contract is the verdict, which stores and writes are +refused; the text is informational. On a load TS judges record by record in file order and names the first collision +it meets, where C# judges the whole store, so the two runtimes can name different pairs of one store. A blank value (absent, empty, or only whitespace) is +never indexed and never conflicts. Two types may share an alias and a value. Index values are compared as the C# +reference renders them: strings as is (never trimmed), `True`/`False`, integers as digits, a non-integral number as +its exact decimal expansion; NaN and infinities have no value. `roles` name selection and reference aliases (the +member's catalog `indexAlias`, so the catalog bytes are the same either way); they imply nothing about uniqueness and +are never indexed for lookup, so `getByIndex` on a role answers `undefined`. An alias is declared in one of the two. + +### Migrating an existing `indexes` key + +Apply per key, and do it BEFORE the CultLib checkout your tools load is rebuilt or updated. A consumer that loads +CultLib at run time (Bifrost reads `$VOIDBOT_CULTLIB_ROOT`, else the sibling `../CultLib` dist) has no pin to hold: its +next start after the checkout moves is the bump. A consumer that pins a version applies this before it bumps the pin. +Moving an `indexes` key to `roles` is safe on the old CultLib too when the definition lists `members` explicitly (an old +codec definition without `members` derives `indexAlias` from `indexes` only, so a moved key would lose its catalog alias), +so the consumer edit lands first: + +1. Nothing looks records up by the key (`getByIndex`, `getKeyByIndex`, `getIdByIndex`; Python `get_by_index`, + `get_key_by_index`, `node.get_by_index`, `watch_by_index`; Rust `get_by_index`): it is a role. Move it to `roles`; + where a definition lists `members` explicitly, delete it from `indexes` and keep the member's `indexAlias` (the + catalog is the same). +2. Something looks it up: it stays unique only when two live records can never hold one non-blank value, because the + record key is derived from that value alone, or the value is an outside identity (email, sku, external id). + Enums, statuses, kinds, categories, parent or foreign ids, shared hashes and timestamps are roles; a lookup on + one already answered an arbitrary record, so move it to a selection (`any_of`) or `getAll` plus a filter. +3. Before the checkout moves, open each real `.cc` once with the new definitions. A refusal names both keys and the alias and + writes nothing. + +Misclassifying costs differently each way. A role declared unique: the second record holding a shared value is +refused at commit, and a store already holding one refuses to open, so that store is down until the key is +reclassified (loud, names both keys and the alias). A unique key declared a role: lookups by it answer nothing and +duplicates stop being refused. + +The Bifrost TS stores (at 781e6ea) call no lookup anywhere, so by rule 1 every key is a role and no key becomes unique. +All six definitions list `members` explicitly, each with its `indexAlias`, so the edit is deleting each `indexes` block: +the catalog is unchanged and the edit is safe on the CultLib it loads today. It must land before that checkout is +updated: + +| Definition | Keys | Verdict | +|---|---|---| +| `update-request` (`tools/agent-transport.mjs`) | `targetRepoName`, `targetAgentIdentity`, `status` | role: many requests per repo and agent; status is an enum | +| `crossing_receipt` (`tools/bifrost-crossing-documents.mjs`) | `commandId`, `crossingKind`, `status` | role: receipts are `crossing__`, several per command; the rest are enums | +| release authority (`tools/bifrost-repository-release-authority.mjs`) | `repositoryFullName`, `upstreamRef`, `commitSha`, `status` | role: the authority id is derived from repo, ref and sha, so one sha under two refs is two records | +| `topic` (`tools/governance-threads.mjs`) | `jurisdictionRepoName`, `jurisdictionAgentIdentity`, `status` | role: many topics per jurisdiction | +| `comment` (`tools/governance-threads.mjs`) | `topicId`, `authorId`, `stance` | role: many comments per topic; stance is an enum | +| `advertisement` (`tools/provider-advertisement.mjs`) | `serviceName` | role: one record holds it today, but nothing looks it up | + You can also register them later if you feel like changing the machine after boot: ```ts await cache.registerNameLookup(playerDocument, "displayName"); -await cache.registerIndex(playerDocument, "faction", "faction"); +await cache.registerIndex(playerDocument, "email", "email"); ``` Registration, `addBackingStore`, `pullAllBackingStores` and every write and @@ -357,7 +419,7 @@ That is the TypeScript version of the original CultCache trick: - the top-level MessagePack value is `[formatVersion, schemaCatalog, records]` - each record is `[key, schemaId, storedAt, payload]` - each catalog entry is `[schemaId, schemaName, schemaVersion, contentHash, canonicalSchemaJson, compatibleSchemaIds, members]` -- it still reads the older TS/Rust-style envelope array and rewrites repaired legacy payloads as v1 snapshots +- it still reads the older TS/Rust-style envelope array (object payloads are normalized in memory; opening never rewrites the file) and replaces it with a v1 snapshot on the next push, delete or flush - it queues local writes in order, then takes the shared `.lock` (`flock` on Unix, `LockFileEx` on Windows, via koffi, loaded on the first write) for each one, so C#, Rust and TypeScript writers exclude each other; the read a write merges from is taken under that lock - it fsyncs the staged file and, on Unix, the directory after the replace; on Windows it replaces with a POSIX-semantics rename and falls back to a write-through `MoveFileExW`, retrying briefly on sharing violations - it persists raw MessagePack payload bytes inside schema-addressed records diff --git a/packages/cultcache-ts/src/cult-cache.ts b/packages/cultcache-ts/src/cult-cache.ts index 88ab94e9..c5bb663d 100644 --- a/packages/cultcache-ts/src/cult-cache.ts +++ b/packages/cultcache-ts/src/cult-cache.ts @@ -1,4 +1,5 @@ import { + declaresRole, decodeDocumentPayload, documentPayloadBytes, documentPayloadFromBytes, @@ -46,6 +47,7 @@ type EntryLookups = { }; type HydratedEntry = CultCacheEnvelope & { + catalogEntry: NonNullable; value: unknown; lookups: EntryLookups; }; @@ -224,7 +226,7 @@ export class CultCache { return this.#serial(() => { const registered = this.#requireDefinition(definition); const nameAccessor = this.#compileAccessor(definition.type, "name", accessor); - const derived = this.#deriveTypeLookups(definition.type, nameAccessor, registered.indexAccessors); + const derived = this.#deriveTypeLookups(definition.type, registered.catalogEntry.schemaName, nameAccessor, registered.indexAccessors); this.#installAccessors(registered, nameAccessor, registered.indexAccessors, derived); return definition; }); @@ -240,10 +242,14 @@ export class CultCache { throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`); } + if (declaresRole(definition, indexName)) { + throw new Error(`Index "${indexName}" of ${definition.type} is declared as a role; a role is never a unique index.`); + } + const registered = this.#requireDefinition(definition); const indexAccessors = new Map(registered.indexAccessors); indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor)); - const derived = this.#deriveTypeLookups(definition.type, registered.nameAccessor, indexAccessors); + const derived = this.#deriveTypeLookups(definition.type, registered.catalogEntry.schemaName, registered.nameAccessor, indexAccessors); this.#installAccessors(registered, registered.nameAccessor, indexAccessors, derived); return definition; }); @@ -311,6 +317,16 @@ export class CultCache { } } + // Judged whole, once: the refusal names the least violation of the state this load would leave, whatever the + // order of the stores and of the entries in them. + refuseLeastIndexSharing(function* () { + for (const entry of next.entries.values()) { + for (const [alias, value] of entry.lookups.indexes) { + yield { type: entry.type, schemaName: entry.catalogEntry.schemaName, alias, value, key: entry.key }; + } + } + }()); + this.#hydrated = next; }); } @@ -418,6 +434,7 @@ export class CultCache { return key ? this.get(definition, key) : undefined; } + // A unique index: one key per non-blank value. A roles alias is never indexed and answers undefined. getKeyByIndex( definition: TDefinition, indexName: string, @@ -579,7 +596,7 @@ export class CultCache { if (existingBySchemaName && existingBySchemaName !== registered) { throw refuse(existingBySchemaName, registered.catalogEntry, `schema name "${registered.catalogEntry.schemaName}"`); } - const derived = this.#deriveTypeLookups(definition.type, nameAccessor, indexAccessors); + const derived = this.#deriveTypeLookups(definition.type, registered.catalogEntry.schemaName, nameAccessor, indexAccessors); // Every check and accessor has run; install. this.#definitions.set(definition.type, registered); @@ -646,6 +663,7 @@ export class CultCache { async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise { const home = this.#homeStore(entry.type); const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value); + this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups); const legacy = this.#hydrated.legacyGlobals.get(entry.type); await home?.push(entry); if (home && legacy) { @@ -795,11 +813,13 @@ export class CultCache { switch (typeof value) { case "string": - return value; - case "number": + return isBlankIndexText(value) ? undefined : value; case "boolean": + return value ? "True" : "False"; case "bigint": - return String(value); + return value.toString(); + case "number": + return canonicalNumberText(value); default: throw new Error(`CultCache ${label} accessor for type "${type}" produced a non-scalar value.`); } @@ -894,6 +914,7 @@ export class CultCache { } } + const lookups = this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value); return { ...entry, payload: this.#cloneBytes(entry.payload), @@ -902,10 +923,22 @@ export class CultCache { schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry, value, - lookups: this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value), + lookups, }; } + // The one index rule (variants:ruling:unique-indexes): within one type and one unique alias, no two records hold one + // non-blank value. The held key at the landing record's own key is its previous self and keeps its value. The message + // names both keys, ordinal, and the alias, never the value. + #refuseIndexSharing(state: HydratedState, type: string, schemaName: string, key: string, lookups: EntryLookups): void { + for (const alias of [...lookups.indexes.keys()].sort(compareOrdinal)) { + const held = state.indexToKeyMaps.get(this.#indexId(type, alias))?.get(lookups.indexes.get(alias)!); + if (held !== undefined && held !== key) { + throw indexSharingRefusal(held, key, schemaName, alias); + } + } + } + // Infallible half of admission: replaces any held record at the same key and installs the admitted one. #installEntry(state: HydratedState, registered: RegisteredDefinition, hydrated: HydratedEntry): void { const entryId = this.#entryId(hydrated.type, hydrated.key); @@ -966,6 +999,7 @@ export class CultCache { // Runs the candidate accessors over every held record of the type into local maps. #deriveTypeLookups( type: string, + schemaName: string, nameAccessor: Accessor | undefined, indexAccessors: ReadonlyMap, ): DerivedTypeLookups { @@ -983,10 +1017,18 @@ export class CultCache { nameMap?.set(lookups.name, entry.key); } for (const [indexName, indexValue] of lookups.indexes) { - indexMaps.get(indexName)?.set(indexValue, entry.key); + const map = indexMaps.get(indexName)!; + if (!map.has(indexValue)) map.set(indexValue, entry.key); } } + // Judged whole, by the same judge as a load: the least violation of the state the registration would leave. + refuseLeastIndexSharing(function* () { + for (const [entry, lookups] of entryLookups) { + for (const [alias, value] of lookups.indexes) yield { type, schemaName, alias, value, key: entry.key }; + } + }()); + return { entryLookups, nameMap, indexMaps }; } @@ -1057,3 +1099,70 @@ export class CultCache { return documentPayloadFromBytes(new Uint8Array(documentPayloadBytes(payload))); } } + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +// The whole-state judge (variants:question:unique-index-refusal-pair-rule): over every (type, alias, value) held at two +// or more distinct keys, throws for the one least by (alias, least key, second-least key, schema name), each compared by +// UTF-16 code unit. A function of the state, never of the order the rows arrive in. +function refuseLeastIndexSharing(rows: Iterable<{ type: string; schemaName: string; alias: string; value: string; key: string }>): void { + const groups = new Map(); + for (const { type, schemaName, alias, value, key } of rows) { + const id = JSON.stringify([type, alias, value]); + const group = groups.get(id); + if (group) group.keys.push(key); + else groups.set(id, { schemaName, alias, keys: [key] }); + } + let least: { schemaName: string; alias: string; first: string; second: string } | undefined; + for (const { schemaName, alias, keys } of groups.values()) { + if (keys.length < 2) continue; + keys.sort(compareOrdinal); + const candidate = { schemaName, alias, first: keys[0], second: keys[1] }; + if (!least || compareLeast(candidate, least) < 0) least = candidate; + } + if (least) throw indexSharingRefusal(least.first, least.second, least.schemaName, least.alias); +} + +function compareLeast( + a: { schemaName: string; alias: string; first: string; second: string }, + b: { schemaName: string; alias: string; first: string; second: string }, +): number { + return compareOrdinal(a.alias, b.alias) || compareOrdinal(a.first, b.first) || compareOrdinal(a.second, b.second) || compareOrdinal(a.schemaName, b.schemaName); +} + +function indexSharingRefusal(keyA: string, keyB: string, schemaName: string, alias: string): Error { + const [first, second] = compareOrdinal(keyA, keyB) <= 0 ? [keyA, keyB] : [keyB, keyA]; + return new Error(`Records ${first} and ${second} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`); +} + +// Blank is no value: empty or only C# White_Space characters. Not JavaScript's \s, which counts U+FEFF and omits U+0085. +const BLANK_INDEX_TEXT = /^[\t-\r \u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000]*$/; + +function isBlankIndexText(value: string): boolean { + return BLANK_INDEX_TEXT.test(value); +} + +// The canonical text of a number, as C# RenderCanonicalNumber: integers as their digits (-0 is 0), a non-integral finite +// double as its exact decimal expansion, NaN and infinities as no value. +function canonicalNumberText(value: number): string | undefined { + if (!Number.isFinite(value)) { + return undefined; + } + if (Number.isInteger(value)) { + return BigInt(value).toString(); + } + + // value = mantissa * 2^exponent with exponent < 0, so value = mantissa * 5^k / 10^k for k = -exponent. + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, value); + const bits = view.getBigUint64(0); + const biasedExponent = Number((bits >> 52n) & 0x7ffn); + const fraction = bits & 0xfffffffffffffn; + const mantissa = biasedExponent === 0 ? fraction : fraction | (1n << 52n); + const k = -((biasedExponent === 0 ? 1 : biasedExponent) - 1075); + const digits = (mantissa * 5n ** BigInt(k)).toString().padStart(k + 1, "0"); + const text = `${digits.slice(0, digits.length - k)}.${digits.slice(digits.length - k)}`.replace(/0+$/, "").replace(/\.$/, ""); + return value < 0 ? `-${text}` : text; +} diff --git a/packages/cultcache-ts/src/document.ts b/packages/cultcache-ts/src/document.ts index 99ec170c..10af96a6 100644 --- a/packages/cultcache-ts/src/document.ts +++ b/packages/cultcache-ts/src/document.ts @@ -32,23 +32,22 @@ export function defineDocumentType( throw new Error(`CultCache document type "${definition.type}" declares an invalid name accessor.`); } - if (Array.isArray(definition.indexes)) { - const seen = new Set(); - for (const index of definition.indexes) { - if (!index.name || index.name.trim().length === 0) { - throw new Error(`CultCache document type "${definition.type}" declares an index with an empty name.`); - } - if (seen.has(index.name)) { - throw new Error(`CultCache document type "${definition.type}" declares duplicate index "${index.name}".`); - } - seen.add(index.name); - } - } else if (definition.indexes) { - for (const name of Object.keys(definition.indexes)) { + const declared = new Set(); + for (const [kind, set] of [["index", definition.indexes], ["role", definition.roles]] as const) { + const names = new Set(); + for (const name of Array.isArray(set) ? set.map((entry) => entry.name) : set ? Object.keys(set) : []) { if (!name || name.trim().length === 0) { - throw new Error(`CultCache document type "${definition.type}" declares an index with an empty name.`); + throw new Error(`CultCache document type "${definition.type}" declares ${kind === "index" ? "an index" : "a role"} with an empty name.`); } + if (names.has(name)) { + throw new Error(`CultCache document type "${definition.type}" declares duplicate ${kind} "${name}".`); + } + if (kind === "role" && declared.has(name)) { + throw new Error(`CultCache document type "${definition.type}" declares "${name}" as both an index and a role; an index already names its role.`); + } + names.add(name); } + for (const name of names) declared.add(name); } if (definition.members) { @@ -132,16 +131,25 @@ export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition return documentPayloadFromBytes(bytes); } +// Whether the definition declares `name` as a role alias (either `roles` form). +export function declaresRole(definition: AnyCultCacheDocumentDefinition, name: string): boolean { + const roles = definition.roles; + return Array.isArray(roles) ? roles.some((entry) => entry.name === name) : roles ? Object.hasOwn(roles, name) : false; +} + // The schema members of a document: the ones it declares, else those its codec derives, else none. export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinition): readonly CultCacheSchemaCatalogMember[] { if (definition.members || !definition.codec) { return definition.members ?? []; } const codec = definition.codec; - const indexAlias = (slotName: string) => - !Array.isArray(definition.indexes) && definition.indexes - ? Object.entries(definition.indexes).find(([, accessor]) => accessor === slotName)?.[0] ?? null - : null; + const indexAlias = (slotName: string) => { + for (const set of [definition.indexes, definition.roles]) { + const alias = !Array.isArray(set) && set ? Object.entries(set).find(([, accessor]) => accessor === slotName)?.[0] : undefined; + if (alias !== undefined) return alias; + } + return null; + }; return codec.members.map((member, index) => ({ ...member, isName: (definition.name as unknown) === codec.names[index], diff --git a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts index 2501c632..6428ef48 100644 --- a/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts +++ b/packages/cultcache-ts/src/single-file-messagepack-backing-store.ts @@ -47,21 +47,11 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { } async pullAll(): Promise { - // The disk decides the header: a file that is gone or legacy is not marked. - this.#format = STORE_FORMAT_VERSION; + // A pure read: the disk decides the header, and a legacy file is normalized in memory only. The file is rewritten + // as v1 by the next push, delete or flush, which merges what this read returns; opening never replaces bytes the + // consumer's codec has not yet accepted. const disk = this.#readDisk(); this.#format = disk.format; - if (disk.repairedLegacyPayload) { - // The heal decides from its own read, taken under the store lock. - await this.#enqueue(async () => { - const again = this.#readDisk(); - this.#format = again.format; - if (again.repairedLegacyPayload) { - await this.#writeAll([], again.envelopes); - } - }); - } - return disk.envelopes; } @@ -158,7 +148,7 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore { export function readSingleFileStore(path: string): SingleFileStoreRead { const data = readStoreBytes(path); if (data === undefined) { - return { absent: true, format: STORE_FORMAT_VERSION, envelopes: [], repairedLegacyPayload: false }; + return { absent: true, format: STORE_FORMAT_VERSION, envelopes: [] }; } try { @@ -395,8 +385,6 @@ export type SingleFileStoreRead = { absent: boolean; format: StoreFormat; envelopes: CultCacheEnvelope[]; - // A legacy file whose payloads were not bytes: reading it repairs them, and the store rewrites the file. - repairedLegacyPayload: boolean; }; function decodeStoreFile(data: Uint8Array): Omit { @@ -420,7 +408,7 @@ function decodeStoreFile(data: Uint8Array): Omit storedAt: record.storedAt, }; }); - return { format: snapshot.format, envelopes, repairedLegacyPayload: false }; + return { format: snapshot.format, envelopes }; } const legacy = decodeLegacyEnvelopeArray(decoded); @@ -428,19 +416,8 @@ function decodeStoreFile(data: Uint8Array): Omit throw new Error("CultCache file is not a recognized CultCache MessagePack store."); } - let repairedLegacyPayload = false; - const normalized = legacy.map((entry) => { - const payload = normalizePayload(entry.payload); - if (payload !== entry.payload) { - repairedLegacyPayload = true; - } - - return { - ...entry, - payload, - }; - }); - return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[], repairedLegacyPayload }; + const normalized = legacy.map((entry) => ({ ...entry, payload: normalizePayload(entry.payload) })); + return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[] }; } function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined { diff --git a/packages/cultcache-ts/src/types.ts b/packages/cultcache-ts/src/types.ts index 88c20f2e..6f6c0923 100644 --- a/packages/cultcache-ts/src/types.ts +++ b/packages/cultcache-ts/src/types.ts @@ -59,9 +59,21 @@ export interface CultCacheDocumentDefinition>; + /** + * Unique lookup aliases. A non-blank value names one record of the type: a write that would leave two records holding + * one value, and a store that already holds two, is refused naming both keys and the alias. Blank values (absent, + * empty, or only whitespace) are never indexed. `getByIndex` and `getKeyByIndex` answer from these. + */ indexes?: | Readonly>>> | readonly CultCacheDocumentIndexDefinition>[]; + /** + * Selection and reference aliases: the member's catalog `indexAlias`. A role implies nothing about uniqueness, many + * records may share a value, and it is never indexed for lookup. An alias is declared in `indexes` or `roles`, not both. + */ + roles?: + | Readonly>>> + | readonly CultCacheDocumentIndexDefinition>[]; formatter?: CultCacheDocumentFormatter>; codec?: SlotsCodec & object>; } diff --git a/packages/cultcache-ts/test/cult-cache.test.ts b/packages/cultcache-ts/test/cult-cache.test.ts index 02324273..de0cad79 100644 --- a/packages/cultcache-ts/test/cult-cache.test.ts +++ b/packages/cultcache-ts/test/cult-cache.test.ts @@ -504,46 +504,56 @@ test("SingleFileMessagePackBackingStore refuses a record whose schema the catalo await assert.rejects(() => cache.pullAllBackingStores(), StoreUnreadableError); }); -test("SingleFileMessagePackBackingStore heals legacy envelopes whose payload was persisted as an object", async () => { - const noteDocument = defineDocumentType({ - type: "note", - schema: z.object({ - title: z.string(), - body: z.string(), - }), - }); +const legacyNoteType = defineDocumentType({ + type: "note", + schema: z.object({ title: z.string(), body: z.string().optional() }), +}); +// SYNTHETIC pre-v1 file (no real one is known): an envelope array with object payloads. +async function legacyNoteStore(second: unknown): Promise<{ storePath: string; bytes: Uint8Array }> { const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "legacy.msgpack"); - const envelopePayload = { - title: "Hello", - body: "world", - }; - - await writeFile( - storePath, - encode([ - { - key: "note:hello", - type: "note", - payload: envelopePayload, - storedAt: new Date().toISOString(), - }, - ]), - ); + const bytes = encode([ + { key: "note:hello", type: "note", payload: { title: "Hello", body: "world" }, storedAt: "2026-10-20T00:00:00Z" }, + { key: "note:other", type: "note", payload: second, storedAt: "2026-10-20T00:00:00Z" }, + ]); + await writeFile(storePath, bytes); + return { storePath, bytes }; +} +test("a codec that refuses a synthetic pre-v1 store's content leaves the file byte-identical", async () => { + const { storePath, bytes } = await legacyNoteStore({ title: 42 }); const cache = CultCache.builder() - .withDocumentType(noteDocument) + .withDocumentType(legacyNoteType) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + + await assert.rejects(() => cache.pullAllBackingStores()); + assert.deepEqual([...(await readFile(storePath))], [...bytes]); + assert.deepEqual((await readdir(dirname(storePath))).sort(), ["legacy.msgpack"]); +}); + +test("a synthetic pre-v1 store is read untouched and becomes v1 only when the consumer writes, keeping its other entries", async () => { + const { storePath, bytes } = await legacyNoteStore({ title: "Other" }); + const cache = CultCache.builder() + .withDocumentType(legacyNoteType) .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) .build(); await cache.pullAllBackingStores(); - assert.deepEqual(cache.getRequired(noteDocument, "note:hello"), envelopePayload); + assert.deepEqual(cache.getRequired(legacyNoteType, "note:hello"), { title: "Hello", body: "world" }); + assert.deepEqual([...(await readFile(storePath))], [...bytes], "reading does not rewrite the file"); - const rewritten = decode(await readFile(storePath)) as unknown[]; - assert.equal(rewritten[0], "cultcache.store.v1"); - const records = rewritten[2] as unknown[][]; - assert.equal(records.length, 1); - assert.ok(records[0]?.[3] instanceof Uint8Array); + await cache.put(legacyNoteType, "note:new", { title: "New" }); + assert.equal((decode(await readFile(storePath)) as unknown[])[0], "cultcache.store.v1"); + + const reopened = CultCache.builder() + .withDocumentType(legacyNoteType) + .withGenericStore(new SingleFileMessagePackBackingStore(storePath)) + .build(); + await reopened.pullAllBackingStores(); + assert.deepEqual(reopened.getRequired(legacyNoteType, "note:hello"), { title: "Hello", body: "world" }); + assert.deepEqual(reopened.getRequired(legacyNoteType, "note:other"), { title: "Other" }); + assert.deepEqual(reopened.getRequired(legacyNoteType, "note:new"), { title: "New" }); }); test("CultCache rejects a second generic backing store", async () => { diff --git a/packages/cultcache-ts/test/first-call.test.ts b/packages/cultcache-ts/test/first-call.test.ts index 3c0e22e5..45f6edd2 100644 --- a/packages/cultcache-ts/test/first-call.test.ts +++ b/packages/cultcache-ts/test/first-call.test.ts @@ -24,7 +24,6 @@ const itemDocument = defineDocumentType({ schemaVersion: "gamecult.first_call_item.v1", schema: z.object({ name: z.string(), category: z.string(), value: z.number().int() }), name: "name", - indexes: { category: "category" }, }); const registry = defineDocumentRegistry(itemDocument); diff --git a/packages/cultcache-ts/test/store-file.test.ts b/packages/cultcache-ts/test/store-file.test.ts index e98309f1..39ac553f 100644 --- a/packages/cultcache-ts/test/store-file.test.ts +++ b/packages/cultcache-ts/test/store-file.test.ts @@ -471,44 +471,70 @@ test("the store lock is released when the operation throws", async () => { } }); +// SYNTHETIC pre-v1 vector: no real file of this shape is known to exist. A top-level array of {key,type,payload,storedAt} +// whose payload is not binary (an object, a Node Buffer JSON, an int array) is what the removed heal-on-open rewrote. const legacyObjectPayload = (): Uint8Array => encode([{ key: "note:hello", type: "note", payload: { title: "Hello" }, storedAt: "2026-10-20T00:00:00Z" }]); -test("an open that heals a legacy store waits for the store lock and heals from its own read", async () => { +test("opening a synthetic pre-v1 store with object payloads returns them normalized and leaves the file untouched, even while another holder has the lock", async () => { const directory = await scratch(); try { const path = join(directory, "legacy.cc"); const legacy = legacyObjectPayload(); await writeFile(path, legacy); const store = new SingleFileMessagePackBackingStore(path); - let healed: Promise | undefined; - let settled = false; await withStoreWriteLock(path, async () => { - healed = store.pullAll().then(() => { - settled = true; - }); - await sleepFor(300); - assert.equal(settled, false, "the heal must wait for the lock"); - assert.deepEqual([...(await readFile(path))], [...legacy], "the file keeps its legacy bytes while the lock is held elsewhere"); + const [lockHeld] = await Promise.all([readdir(directory)]); + const envelopes = await within(store.pullAll(), 5000, "an open does not wait for the write lock"); + assert.deepEqual(envelopes.map((entry) => entry.key), ["note:hello"]); + assert.deepEqual(decode(documentPayloadBytes(envelopes[0].payload)), { title: "Hello" }); + assert.deepEqual([...(await readFile(path))], [...legacy]); + assert.deepEqual((await readdir(directory)).sort(), lockHeld.sort(), "an open adds no lock or temp file"); }); - await within(healed!, 5000, "the heal"); - assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1"); + assert.deepEqual([...(await readFile(path))], [...legacy]); + assert.deepEqual((await readdir(directory)).filter((name) => name !== "legacy.cc" && name !== "legacy.cc.lock"), []); } finally { await rm(directory, { recursive: true, force: true }); } }); -test("push and delete over a legacy store that needs healing finish instead of waiting on their own lock", async () => { +test("opening a synthetic pre-v1 store with Buffer-JSON and int-array payloads normalizes them in memory and leaves the file untouched", async () => { + const directory = await scratch(); + try { + const path = join(directory, "legacy.cc"); + const inner = encode({ title: "Hello" }); + const legacy = encode([ + { key: "a", type: "note", payload: { type: "Buffer", data: [...inner] }, storedAt: "2026-10-20T00:00:00Z" }, + { key: "b", type: "note", payload: [...inner], storedAt: "2026-10-20T00:00:00Z" }, + ]); + await writeFile(path, legacy); + const envelopes = await new SingleFileMessagePackBackingStore(path).pullAll(); + assert.deepEqual(envelopes.map((entry) => entry.key), ["a", "b"]); + for (const envelope of envelopes) { + assert.deepEqual(decode(documentPayloadBytes(envelope.payload)), { title: "Hello" }); + } + assert.deepEqual([...(await readFile(path))], [...legacy]); + assert.deepEqual(await readdir(directory), ["legacy.cc"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("push and delete over a synthetic pre-v1 store write v1 and keep the other legacy entries, normalized payloads included", async () => { const directory = await scratch(); try { const path = join(directory, "legacy.cc"); await writeFile(path, legacyObjectPayload()); const store = new SingleFileMessagePackBackingStore(path); await within(store.push(entryOf("a")), 5000, "push"); + assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1"); + assert.deepEqual((await store.pullAll()).map((entry) => entry.key).sort(), ["a", "note:hello"]); await writeFile(path, legacyObjectPayload()); await within(store.delete(entryOf("a")), 5000, "delete"); - const keys = (await store.pullAll()).map((entry) => entry.key); - assert.deepEqual(keys, ["note:hello"]); + assert.equal((decode(await readFile(path)) as unknown[])[0], "cultcache.store.v1"); + const kept = await store.pullAll(); + assert.deepEqual(kept.map((entry) => entry.key), ["note:hello"]); + assert.deepEqual(decode(documentPayloadBytes(kept[0].payload)), { title: "Hello" }); } finally { await rm(directory, { recursive: true, force: true }); } @@ -697,7 +723,7 @@ test("a store file of an unrecognized shape is refused, and a store naming a mis } }); -test("opening a store that needs no healing writes nothing: a missing store leaves no lock file, a legacy store with byte payloads keeps its bytes", async () => { +test("opening writes nothing: a missing store leaves no lock file, a legacy store with byte payloads keeps its bytes", async () => { const directory = await scratch(); try { const absent = join(directory, "nested", "absent.cc"); diff --git a/packages/cultcache-ts/test/unique-index.test.ts b/packages/cultcache-ts/test/unique-index.test.ts new file mode 100644 index 00000000..d36c2cf2 --- /dev/null +++ b/packages/cultcache-ts/test/unique-index.test.ts @@ -0,0 +1,583 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { copyFile, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { test } from "node:test"; + +import { decode } from "@msgpack/msgpack"; +import { z } from "zod"; + +import { codecs } from "../src"; +import { CultCache } from "../src/cult-cache"; +import { defineDocumentType, resolveDocumentMembers } from "../src/document"; +import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store"; +import type { AnyCultCacheDocumentDefinition } from "../src/types"; + +// variants:ruling:unique-indexes. `indexes` are unique within one type, `roles` are selection aliases. The vectors in +// tests/vectors/unique-index are written by the C# reference; the rest of the behaviour is pinned here. + +function findVectors(from: string): string { + for (let dir = from; ; dir = resolve(dir, "..")) { + if (existsSync(join(dir, "tests", "vectors", "unique-index"))) return join(dir, "tests", "vectors", "unique-index"); + if (resolve(dir, "..") === dir) throw new Error("tests/vectors/unique-index not found above the test."); + } +} +const vectors = findVectors(__dirname); + +const CANARY = "CANARY-uidx-5e1b"; + +const skuSchema = z.object({ sku: z.any().optional(), group: z.string().optional() }); + +function item(type: string, where: "indexes" | "roles" | "none" = "indexes", schemaName = `tests.${type}`) { + return defineDocumentType({ + type, + schemaId: `${schemaName}.v1`, + schemaName, + schemaVersion: `${schemaName}.v1`, + schema: skuSchema, + ...(where === "none" ? {} : { [where]: { sku: "sku" } }), + }); +} + +// A definition whose payload is never read back, so a value of any scalar kind (a bigint included) can be held and indexed. +function scalar(type: string) { + const held: unknown[] = []; + return defineDocumentType({ + type, + schemaId: `tests.${type}.v1`, + schemaName: `tests.${type}`, + schemaVersion: `tests.${type}.v1`, + schema: skuSchema, + formatter: { + encode: (value: unknown) => new Uint8Array([held.push(value) - 1]), + decode: (payload: Uint8Array) => held[payload[0]] as never, + }, + indexes: { sku: "sku" }, + }); +} + +function memory(...definitions: AnyCultCacheDocumentDefinition[]) { + let builder = CultCache.builder(); + for (const definition of definitions) builder = builder.withDocumentType(definition); + return builder.build(); +} + +function refusal(keyA: string, keyB: string, schemaName: string, alias: string) { + return `Records ${keyA} and ${keyB} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`; +} + +test("a put of a held value at another key is refused naming both keys and the alias, and nothing moves", async () => { + const dir = await mkdtemp(join(tmpdir(), "uidx-")); + try { + const file = join(dir, "items.cc"); + const document = item("item"); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + await cache.pullAllBackingStores(); + await cache.put(document, "zeta", { sku: CANARY }); + const before = await readFile(file); + const snapshot = cache.snapshot(); + + await assert.rejects(cache.put(document, "alpha", { sku: CANARY }), (error: Error) => { + assert.equal(error.message, refusal("alpha", "zeta", "tests.item", "sku")); + assert.ok(!error.message.includes(CANARY), "the value is never echoed"); + return true; + }); + assert.deepEqual(await readFile(file), before, "the store is byte-identical"); + assert.deepEqual(cache.snapshot(), snapshot); + assert.equal(cache.get(document, "alpha"), undefined); + assert.equal(cache.getKeyByIndex(document, "sku", CANARY), "zeta"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("rewriting a record's own key with its own value lands", async () => { + const document = item("item"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S", group: "one" }); + await cache.put(document, "a", { sku: "S", group: "two" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "a"); + assert.equal(cache.get(document, "a")?.group, "two"); +}); + +test("a removed or changed holder releases its value", async () => { + const document = item("item"); + const cache = memory(document); + await cache.put(document, "b", { sku: "S" }); + await assert.rejects(cache.put(document, "a", { sku: "S" }), /Records a and b/u); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "b", "the refused write did not take the value"); + + await cache.delete(document, "b"); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined); + await cache.put(document, "a", { sku: "S" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "a"); + + await cache.put(document, "a", { sku: "T" }); + await cache.put(document, "b", { sku: "S" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "b"); + assert.equal(cache.getKeyByIndex(document, "sku", "T"), "a"); +}); + +// The C# White_Space set, and nothing else, is blank. Every UTF-16 code unit is judged, so a rule written for a few +// characters (or for JavaScript's \s, which counts U+FEFF and omits U+0085) cannot pass. +const CSHARP_WHITE_SPACE = (code: number) => + (code >= 0x09 && code <= 0x0d) || code === 0x20 || code === 0x85 || code === 0xa0 || code === 0x1680 || + (code >= 0x2000 && code <= 0x200a) || code === 0x2028 || code === 0x2029 || code === 0x202f || code === 0x205f || code === 0x3000; + +test("blank is exactly the C# White_Space set: every code unit, alone, repeated and padded", async () => { + const document = item("item"); + const cache = memory(document); + const disagreements: string[] = []; + for (let code = 0; code <= 0xffff; code++) { + if (code >= 0xd800 && code <= 0xdfff) continue; // lone surrogates do not survive MessagePack text + const character = String.fromCharCode(code); + await cache.put(document, "x", { sku: character }); + let blank = true; + try { + await cache.put(document, "y", { sku: character }); + } catch { + blank = false; + } + if (blank !== CSHARP_WHITE_SPACE(code)) disagreements.push(code.toString(16)); + await cache.delete(document, "x"); + await cache.delete(document, "y"); + } + assert.deepEqual(disagreements, []); + + // Not trimmed: a padded value is the value, distinct from its trimmed form. + await cache.put(document, "x", { sku: " a " }); + await cache.put(document, "y", { sku: "a" }); + assert.equal(cache.getKeyByIndex(document, "sku", " a "), "x"); + assert.equal(cache.getKeyByIndex(document, "sku", "a"), "y"); + await assert.rejects(cache.put(document, "z", { sku: " a " }), /Records x and z/u); +}); + +test("blank values are never indexed and never conflict: empty, spaces, absent", async () => { + const document = item("item"); + const cache = memory(document); + for (const [key, sku] of [["a", ""], ["b", " "], ["c", "\u0085"], ["d", undefined], ["e", "　"]] as const) { + await cache.put(document, key, { sku }); + } + assert.equal(cache.getKeyByIndex(document, "sku", ""), undefined); + assert.equal(cache.getKeyByIndex(document, "sku", " "), undefined); + // U+FEFF and U+001F are values. + await cache.put(document, "f", { sku: "" }); + await assert.rejects(cache.put(document, "g", { sku: "" }), /Records f and g/u); + await cache.put(document, "h", { sku: "\u001F" }); + await assert.rejects(cache.put(document, "i", { sku: "\u001F" }), /Records h and i/u); +}); + +test("two types may share an alias and a value", async () => { + const first = item("first"); + const second = item("second"); + const cache = memory(first, second); + await cache.put(first, "a", { sku: "S" }); + await cache.put(second, "b", { sku: "S" }); + assert.equal(cache.getKeyByIndex(first, "sku", "S"), "a"); + assert.equal(cache.getKeyByIndex(second, "sku", "S"), "b"); +}); + +test("roles may share values and are not looked up", async () => { + const document = item("item", "roles"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S" }); + await cache.put(document, "b", { sku: "S" }); + await cache.put(document, "c", { sku: "S" }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined); + assert.equal(cache.getByIndex(document, "sku", "S"), undefined); +}); + +test("an alias declared in indexes and roles is refused at definition; each map is checked like indexes", () => { + const base = { type: "bad", schema: skuSchema, schemaId: "tests.bad.v1", schemaName: "tests.bad", schemaVersion: "tests.bad.v1" }; + assert.throws( + () => defineDocumentType({ ...base, indexes: { sku: "sku" }, roles: { sku: "sku" } }), + { message: 'CultCache document type "bad" declares "sku" as both an index and a role; an index already names its role.' }, + ); + assert.throws(() => defineDocumentType({ ...base, roles: { " ": "sku" } }), /declares a role with an empty name/u); + assert.throws( + () => defineDocumentType({ ...base, roles: [{ name: "r", accessor: "sku" }, { name: "r", accessor: "group" }] }), + /declares duplicate role "r"/u, + ); + assert.doesNotThrow(() => defineDocumentType({ ...base, indexes: { sku: "sku" }, roles: { group: "group" } })); +}); + +test("a codec definition's catalog entry is identical with an alias under indexes or roles", async () => { + const codec = codecs.slots<{ sku: string; group: string }>({ 0: ["sku", codecs.string], 1: ["group", codecs.string] }, "T.Item"); + const make = (where: "indexes" | "roles") => + defineDocumentType({ + type: "codec_item", + schemaId: "tests.codec_item.v1", + schemaName: "tests.codec_item", + schemaVersion: "tests.codec_item.v1", + schema: z.object({ sku: z.string(), group: z.string() }), + codec, + [where]: { sku: "sku" }, + }); + const catalog = async (definition: AnyCultCacheDocumentDefinition) => { + const cache = memory(definition); + await cache.put(definition, "a", { sku: "S", group: "g" } as never); + return cache.snapshot()[0].catalogEntry!; + }; + const members = (definition: AnyCultCacheDocumentDefinition) => resolveDocumentMembers(definition); + assert.deepEqual(members(make("indexes")), members(make("roles"))); + assert.equal(members(make("roles")).find((member) => member.memberName === "Sku")?.indexAlias, "sku"); + const [unique, role] = [await catalog(make("indexes")), await catalog(make("roles"))]; + assert.equal(unique.canonicalSchemaJson, role.canonicalSchemaJson); + assert.equal(unique.contentHash, role.contentHash); + assert.equal(unique.schemaId, role.schemaId); +}); + +test("vectors values.msgpack: every row's canonical text, blank rows no value", async () => { + const rows = decode(await readFile(join(vectors, "values.msgpack")), { useBigInt64: true }) as [string, string, unknown, string | null][]; + assert.ok(rows.length >= 20); + for (const [label, kind, value, text] of rows) { + const document = scalar(`value_${label.replaceAll(/[^a-z0-9]/gu, "_")}`); + const cache = memory(document); + void kind; + const normalized = value; + await cache.put(document, "one", { sku: normalized }); + if (text === null) { + await cache.put(document, "two", { sku: normalized }); + assert.equal(cache.get(document, "two") !== undefined, true, `${label}: a blank value commits twice`); + for (const probe of ["", " ", "NaN", "0", "Infinity", "-Infinity", "null", "undefined"]) { + assert.equal(cache.getKeyByIndex(document, "sku", probe), undefined, `${label}: found by '${probe}'`); + } + } else { + assert.equal(cache.getKeyByIndex(document, "sku", text), "one", `${label}: found by its text`); + await assert.rejects(cache.put(document, "two", { sku: normalized }), /Records one and two/u, label); + } + } +}); + +// An independent oracle for the double rule: the text must denote exactly the double, so N / 10^d == mantissa * 2^exponent +// in exact integer arithmetic, with no trailing zero and no bare point. A rule written to a few spellings cannot pass. +function exactlyDenotes(text: string, value: number): boolean { + const match = /^(-?)(\d+)(?:\.(\d*[1-9]))?$/u.exec(text); + if (!match) return false; + const decimals = match[3]?.length ?? 0; + const numerator = BigInt(match[2] + (match[3] ?? "")); + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, Math.abs(value)); + const bits = view.getBigUint64(0); + const biased = Number((bits >> 52n) & 0x7ffn); + const mantissa = biased === 0 ? bits & 0xfffffffffffffn : (bits & 0xfffffffffffffn) | (1n << 52n); + const exponent = (biased === 0 ? 1 : biased) - 1075; + const lhs = numerator * (exponent >= 0 ? 1n : 2n ** BigInt(-exponent)); + const rhs = mantissa * (exponent >= 0 ? 2n ** BigInt(exponent) : 1n) * 10n ** BigInt(decimals); + return (match[1] === "-") === (value < 0) && lhs === rhs; +} + +test("a number's index text is its exact value: integers as digits, doubles as the exact expansion, NaN and infinities none", async () => { + const document = scalar("number"); + const cache = memory(document); + const probes = [0.1, -0.1, 2.5e-7, 1.5, -123.456, 5e-324, 2.2250738585072014e-308, 1e-10, 123456789.123456789, Number.MIN_VALUE * 3, + ...Array.from({ length: 200 }, (_, i) => Math.sin(i + 1) * 10 ** ((i % 40) - 20)), 1e21 + 2 ** 20 + 0.5 * 0]; + // Round trip through lookups: the exact expansion of each probe is found, and it denotes the probe exactly. + for (const value of probes) { + if (Number.isInteger(value)) continue; + await cache.put(document, "k", { sku: value }); + const candidates = candidateTexts(value); + const found = candidates.filter((candidate) => cache.getKeyByIndex(document, "sku", candidate) === "k"); + assert.equal(found.length, 1, `${value}: exactly one spelling is its index text`); + assert.ok(exactlyDenotes(found[0], value), `${value}: ${found[0]} denotes it exactly`); + } + + await cache.put(document, "k", { sku: 1e21 }); + assert.equal(cache.getKeyByIndex(document, "sku", "1000000000000000000000"), "k"); + await cache.put(document, "k", { sku: -0 }); + assert.equal(cache.getKeyByIndex(document, "sku", "0"), "k"); + await cache.put(document, "k", { sku: 12n }); + assert.equal(cache.getKeyByIndex(document, "sku", "12"), "k"); + await cache.put(document, "k", { sku: true }); + assert.equal(cache.getKeyByIndex(document, "sku", "True"), "k"); + await cache.put(document, "k", { sku: false }); + assert.equal(cache.getKeyByIndex(document, "sku", "False"), "k"); + for (const value of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { + await cache.put(document, "p", { sku: value }); + await cache.put(document, "q", { sku: value }); + } +}); + +// Spellings a lookup could be probed with: the exact decimal expansion computed by an arbitrary-precision route. +function candidateTexts(value: number): string[] { + const view = new DataView(new ArrayBuffer(8)); + view.setFloat64(0, Math.abs(value)); + const bits = view.getBigUint64(0); + const biased = Number((bits >> 52n) & 0x7ffn); + const mantissa = biased === 0 ? bits & 0xfffffffffffffn : (bits & 0xfffffffffffffn) | (1n << 52n); + const k = 1075 - (biased === 0 ? 1 : biased); + const scaled = (mantissa * 5n ** BigInt(k)).toString().padStart(k + 1, "0"); + const point = scaled.length - k; + const text = `${scaled.slice(0, point)}.${scaled.slice(point)}`.replace(/0+$/u, "").replace(/\.$/u, ""); + return [(value < 0 ? "-" : "") + text, String(value), value.toString()].filter((spelling, i, all) => all.indexOf(spelling) === i); +} + +function twoDefinitions(schemaId: string, where: "indexes" | "roles") { + return defineDocumentType({ + type: "unique_item", + schemaId, + schemaName: "tests.unique_item", + schemaVersion: "tests.unique_item.v1", + schema: z.object({ sku: z.string(), category: z.string() }), + codec: codecs.slots<{ sku: string; category: string }>({ 0: ["sku", codecs.string], 1: ["category", codecs.string] }, "UqVecItem"), + [where]: { Sku: "sku" }, + }); +} + +test("vectors duplicate-store.cc and blank-store.cc hold their verdicts", async () => { + const outcomes = decode(await readFile(join(vectors, "outcomes.msgpack"))) as [string, string | null, string, string | null, string | null][]; + assert.equal(outcomes.length, 3); + const dir = await mkdtemp(join(tmpdir(), "uidx-vec-")); + try { + for (const [name, alias, outcome, key1, key2] of outcomes) { + const file = join(dir, `${outcomes.indexOf(outcomes.find((row) => row[0] === name && row[1] === alias)!)}-${name}`); + await copyFile(join(vectors, name), file); + const before = await readFile(file); + const [envelope] = await new SingleFileMessagePackBackingStore(file).pullAll(); + const document = twoDefinitions(envelope.schemaId!, alias === null ? "roles" : "indexes"); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + const stateBefore = cache.snapshot(); + if (outcome === "open") { + await cache.pullAllBackingStores(); + assert.equal(cache.snapshot().length, name === "blank-store.cc" ? 2 : 4); + if (alias === null) assert.equal(cache.getKeyByIndex(document, "Sku", "S-1"), undefined); + } else { + await assert.rejects(cache.pullAllBackingStores(), (error: Error) => { + assert.equal(error.message, refusal(key1!, key2!, "tests.unique_item", alias!)); + assert.ok(!error.message.includes("S-1")); + return true; + }); + assert.deepEqual(cache.snapshot(), stateBefore, "a refused load admits nothing"); + } + assert.deepEqual(await readFile(file), before, `${name} is untouched`); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// refusals.msgpack: rows [label, ops, refusal] written by the C# reference. TypeScript has no batch commit +// (variants:idea:unique-index-runtimes.ts-batch-commit), so it performs every row whose commits each hold one record: +// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 of the README (20). +const REFUSAL_SCHEMAS: [string, string, string, Record][] = [ + ["unique_item", "tests.unique_item", "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", { Sku: "sku" }], + ["uq_pair", "tests.uq_pair", "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", { Sku: "sku", Alt: "second" }], + ["uq_alias", "tests.uq_alias", "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", { Sku: "sku", alt: "second" }], + ["uq_wide", "tests.uq_wide", "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", { "\uFF21": "sku", "\u{10400}": "second" }], + ["uq_alpha", "tests.uq_alpha", "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", { Sku: "sku" }], + ["uq_beta", "tests.uq_beta", "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", { Sku: "sku" }], +]; + +type RefusalOp = ["open", string | null] | ["pull", string] | ["commit", [string, string, string, string][]]; + +test("vectors refusals.msgpack: every row TS can perform", async () => { + const rows = decode(await readFile(join(vectors, "refusals.msgpack"))) as [string, RefusalOp[], [string, string, string, string] | null][]; + const definitions = new Map( + REFUSAL_SCHEMAS.map(([type, schemaName, schemaId, indexes]) => [ + schemaName, + defineDocumentType({ + type, + schemaId, + schemaName, + schemaVersion: `${schemaName}.v1`, + schema: z.object({ sku: z.string(), second: z.string() }), + codec: codecs.slots<{ sku: string; second: string }>({ 0: ["sku", codecs.string], 1: ["second", codecs.string] }, `Uq${type}`), + indexes, + }), + ]), + ); + const dir = await mkdtemp(join(tmpdir(), "uidx-refusals-")); + let walked = 0; + try { + for (const [label, ops, refused] of rows) { + if (ops.some((op) => op[0] === "commit" && op[1].length > 1)) continue; + const file = join(dir, `${walked}.cc`); + let cache: CultCache | undefined; + const perform = async (op: RefusalOp) => { + if (op[0] === "open") { + if (op[1] !== null) await copyFile(join(vectors, op[1]), file); + let builder = CultCache.builder().withGenericStore(new SingleFileMessagePackBackingStore(file)); + for (const definition of definitions.values()) builder = builder.withDocumentType(definition); + cache = builder.build(); + await cache.pullAllBackingStores(); + } else if (op[0] === "pull") { + await copyFile(join(vectors, op[1]), file); + await cache!.pullAllBackingStores(); + } else { + for (const [key, schema, sku, second] of op[1]) await cache!.put(definitions.get(schema)!, key, { sku, second }); + } + }; + for (const op of ops.slice(0, -1)) await perform(op); + const last = ops[ops.length - 1]; + if (refused === null) { + await perform(last); + } else { + const [k1, k2, schema, alias] = refused; + const before = last[0] === "commit" || (last[0] === "open" && last[1] === null) + ? (existsSync(file) ? await readFile(file) : Buffer.alloc(0)) + : await readFile(join(vectors, last[1]!)); + await assert.rejects(perform(last), { message: refusal(k1, k2, schema, alias) }, label); + assert.deepEqual(existsSync(file) ? await readFile(file) : Buffer.alloc(0), before, `${label}: the file is untouched`); + } + walked++; + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + assert.equal(walked, 20); +}); + +test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => { + const dir = await mkdtemp(join(tmpdir(), "uidx-load-")); + try { + const bad = join(dir, "bad.cc"); + await copyFile(join(vectors, "duplicate-store.cc"), bad); + const [envelope] = await new SingleFileMessagePackBackingStore(bad).pullAll(); + const document = twoDefinitions(envelope.schemaId!, "indexes"); + const cache = CultCache.builder().withDocumentType(document).withGenericStore(new SingleFileMessagePackBackingStore(bad)).build(); + await assert.rejects(cache.pullAllBackingStores(), /Records item:a and item:b/u); + assert.equal(cache.get(document, "item:a"), undefined); + assert.equal(cache.getKeyByIndex(document, "Sku", "S-1"), undefined); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("registerIndex refuses a name the definition declares as a role", async () => { + const document = item("item", "roles"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S", group: "g" }); + await cache.put(document, "b", { sku: "S", group: "h" }); + await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: /declared as a role; a role is never a unique index/u }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined); + await cache.put(document, "c", { sku: "S", group: "i" }); +}); + +test("registerIndex over held duplicates installs nothing", async () => { + const document = item("item", "none"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S", group: "g" }); + await cache.put(document, "b", { sku: "S", group: "h" }); + await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: refusal("a", "b", "tests.item", "sku") }); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined, "no map was installed"); + await cache.put(document, "c", { sku: "S", group: "i" }); + await cache.registerIndex(document, "group", "group"); + assert.equal(cache.getKeyByIndex(document, "group", "h"), "b"); + await assert.rejects(cache.put(document, "d", { sku: "x", group: "h" }), /Records b and d/u); +}); + +test("registerIndex over distinct held values installs the lookup", async () => { + const document = item("item", "none"); + const cache = memory(document); + await cache.put(document, "a", { sku: "S" }); + await cache.registerIndex(document, "sku", "sku"); + assert.equal(cache.getKeyByIndex(document, "sku", "S"), "a"); +}); + +// ---- a type with several unique indexes: the second alias is judged, in alias order, in both judges ---- + +const pairSchema = z.object({ a: z.string().optional(), b: z.string().optional() }); + +function pair(where: { indexes?: Record; roles?: Record }) { + return defineDocumentType({ + type: "pair", + schemaId: "tests.pair.v1", + schemaName: "tests.pair", + schemaVersion: "tests.pair.v1", + schema: pairSchema, + ...where, + }); +} + +// Declared b first, so insertion order is the opposite of alias order. +const BOTH_UNIQUE = { indexes: { b: "b", a: "a" } as Record }; + +test("a collision on the second unique alias is refused, with a value on the first alias that does not collide", async () => { + const document = pair(BOTH_UNIQUE); + const cache = memory(document); + await cache.put(document, "k1", { a: "x1", b: "shared" }); + await assert.rejects(cache.put(document, "k2", { a: "x2", b: "shared" }), { message: refusal("k1", "k2", "tests.pair", "b") }); + await assert.rejects(cache.put(document, "k2", { a: "x1", b: "y" }), { message: refusal("k1", "k2", "tests.pair", "a") }); + assert.equal(cache.get(document, "k2"), undefined); +}); + +test("when two aliases collide the first alias in ordinal order is named, and its keys are in ordinal order", async () => { + const document = pair(BOTH_UNIQUE); + const cache = memory(document); + await cache.put(document, "m", { a: "same-a", b: "same-b" }); + await assert.rejects(cache.put(document, "c", { a: "same-a", b: "same-b" }), { message: refusal("c", "m", "tests.pair", "a") }); + // The colliding holders differ per alias: a is held by m, b by n; alias a is still the one named. + await cache.put(document, "n", { a: "other-a", b: "other-b" }); + await assert.rejects(cache.put(document, "z", { a: "same-a", b: "other-b" }), { message: refusal("m", "z", "tests.pair", "a") }); +}); + +test("a load judges every unique alias: a store collision on the second alias refuses the open", async () => { + const dir = await mkdtemp(join(tmpdir(), "uidx-pair-")); + try { + for (const [name, records, expected] of [ + ["second-only", [["k1", { a: "1", b: "s" }], ["k2", { a: "2", b: "s" }]], refusal("k1", "k2", "tests.pair", "b")], + ["both-aliases", [["m", { a: "s", b: "s" }], ["c", { a: "s", b: "s" }]], refusal("c", "m", "tests.pair", "a")], + ] as const) { + const file = join(dir, `${name}.cc`); + const roles = pair({ roles: { b: "b", a: "a" } }); + const writer = CultCache.builder().withDocumentType(roles).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + for (const [key, value] of records) await writer.put(roles, key, value); + const before = await readFile(file); + + const unique = pair(BOTH_UNIQUE); + const reader = CultCache.builder().withDocumentType(unique).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + await assert.rejects(reader.pullAllBackingStores(), { message: expected }, name); + assert.deepEqual(reader.snapshot(), [], "a refused load admits nothing"); + assert.deepEqual(await readFile(file), before); + + const rolesReader = CultCache.builder().withDocumentType(roles).withGenericStore(new SingleFileMessagePackBackingStore(file)).build(); + await rolesReader.pullAllBackingStores(); + assert.equal(rolesReader.snapshot().length, 2, `${name}: the same store opens under roles`); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("registerIndex of a second unique alias judges it, naming the two smallest keys whatever order they were written in", async () => { + const document = pair({ indexes: { a: "a" } }); + const cache = memory(document); + await cache.put(document, "z", { a: "1", b: "s" }); + await cache.put(document, "m", { a: "2", b: "s" }); + await cache.put(document, "q", { a: "3", b: "s" }); + await assert.rejects(cache.registerIndex(document, "b", "b"), { message: refusal("m", "q", "tests.pair", "b") }); + assert.equal(cache.getKeyByIndex(document, "a", "1"), "z", "the alias already installed is intact"); + assert.equal(cache.getKeyByIndex(document, "b", "s"), undefined); +}); + +// ---- the single-put judge's two orders, which no pair-order vector row reaches from TS (a vector row's second record +// arrives in a batch that TS cannot perform): aliases by UTF-16 code unit, then the key pair by UTF-16 code unit ---- + +test("a put colliding on two aliases names the one least by UTF-16 code unit, not by code point", async () => { + // U+10400 is the surrogate pair D801 DC00, so it sorts before U+FF21 by code unit and after it by code point. + const document = pair({ indexes: { "Ａ": "a", "\u{10400}": "b" } }); + const cache = memory(document); + await cache.put(document, "h1", { a: "s1", b: "t1" }); + await cache.put(document, "h2", { a: "s2", b: "t2" }); + await assert.rejects(cache.put(document, "p", { a: "s1", b: "t2" }), { message: refusal("h2", "p", "tests.pair", "\u{10400}") }); + assert.equal(cache.get(document, "p"), undefined); +}); + +test("a put names its key pair by UTF-16 code unit, not by normalised form", async () => { + // "e" U+0302 is the NFD form of U+00EA: equal when normalised, and less by code unit. + const document = item("item"); + const cache = memory(document); + await cache.put(document, "ê", { sku: "v" }); + await assert.rejects(cache.put(document, "eÌ‚", { sku: "v" }), { message: refusal("eÌ‚", "ê", "tests.item", "sku") }); + assert.equal(cache.get(document, "eÌ‚"), undefined); +}); + +test("a put colliding on two aliases orders them by UTF-16 code unit, not by normalised form", async () => { + // U+00EA normalises to e U+0302, which is less than e U+0303 when normalised and greater by code unit. + const document = pair({ indexes: { "ê": "a", "ẽ": "b" } }); + const cache = memory(document); + await cache.put(document, "h1", { a: "s1", b: "t1" }); + await cache.put(document, "h2", { a: "s2", b: "t2" }); + await assert.rejects(cache.put(document, "p", { a: "s1", b: "t2" }), { message: refusal("h2", "p", "tests.pair", "ẽ") }); +}); diff --git a/packages/cultmath/src/CultMath/CultMath.csproj b/packages/cultmath/src/CultMath/CultMath.csproj index ed99c004..d3846d4f 100644 --- a/packages/cultmath/src/CultMath/CultMath.csproj +++ b/packages/cultmath/src/CultMath/CultMath.csproj @@ -10,7 +10,7 @@ GameCult.Math MIT AND MPL-2.0 AND Apache-2.0 - 0.5.0 + 0.5.1 Clean-room C# math shaped for HLSL-style renderer code. GameCult README.md diff --git a/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md b/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md index 7edfa1b2..ebe21e7d 100644 --- a/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md +++ b/packages/cultmath/unity/org.gamecult.cultmath/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this package are documented in this file. Earlier releases (0.2.0-0.2.3) predate this package's changelog and are not backfilled here; see `docs/semver-policy.md` for why. -## [Unreleased] +## [0.5.1] - 2026-11-11 ### Added diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll index 205d3f9b..7757c254 100644 Binary files a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll and b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.dll differ diff --git a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb index 9f865c09..e0991c71 100644 Binary files a/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb and b/packages/cultmath/unity/org.gamecult.cultmath/Runtime/Plugins/CultMath.pdb differ diff --git a/packages/cultmath/unity/org.gamecult.cultmath/package.json b/packages/cultmath/unity/org.gamecult.cultmath/package.json index 20ffe196..e06e6c89 100644 --- a/packages/cultmath/unity/org.gamecult.cultmath/package.json +++ b/packages/cultmath/unity/org.gamecult.cultmath/package.json @@ -1,7 +1,7 @@ { "name": "org.gamecult.cultmath", "displayName": "CultMath", - "version": "0.5.0", + "version": "0.5.1", "unity": "2021.3", "description": "Precompiled CultMath numeric primitives and matching HLSL semantics for GameCult Unity runtimes.", "license": "MIT AND MPL-2.0 AND Apache-2.0", diff --git a/packages/cultmesh-browser/test/content.test.ts b/packages/cultmesh-browser/test/content.test.ts index d4ec4d6e..2f77904e 100644 --- a/packages/cultmesh-browser/test/content.test.ts +++ b/packages/cultmesh-browser/test/content.test.ts @@ -567,8 +567,14 @@ test("a size the runtime cannot allocate is too-large in both exports, not a Ran const count = 2 ** 15; const huge = { ...manifest, sizeBytes: count * int32Max, chunks: Array.from({ length: count }, (_, index) => ({ ...manifest.chunks[0]!, offset: index * int32Max, sizeBytes: int32Max })) }; const server = double(() => { throw new Error("no request expected"); }); - await assert.rejects(fetchVerifiedContent(ENDPOINT, huge, { fetch: server.fetch, maxBytes: size }), codeOf("too-large")); - await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, size, { fetch: server.fetch, maxBytes: size }), codeOf("too-large")); + // a host with overcommit hands out 64 TiB lazily, so the refusal is made certain: the runtime cannot allocate it + const native = globalThis.Uint8Array; + const refusing = new Proxy(native, { construct: (target, args, receiver) => { if (typeof args[0] === "number" && args[0] > 2 ** 40) throw new RangeError("Array buffer allocation failed"); return Reflect.construct(target, args, receiver); } }); + globalThis.Uint8Array = refusing; + try { + await assert.rejects(fetchVerifiedContent(ENDPOINT, huge, { fetch: server.fetch, maxBytes: size }), codeOf("too-large")); + await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, size, { fetch: server.fetch, maxBytes: size }), codeOf("too-large")); + } finally { globalThis.Uint8Array = native; } assert.equal(server.calls.length, 0); }); diff --git a/packages/cultmesh-py/README.md b/packages/cultmesh-py/README.md index 60ae4e49..020a2afc 100644 --- a/packages/cultmesh-py/README.md +++ b/packages/cultmesh-py/README.md @@ -43,9 +43,10 @@ from cultmesh_py import ( note_doc = define_database_entry_type( "mesh.note", - [("body", 0), ("name", 1), ("kind", 2)], + [("body", 0), ("name", 1), ("kind", 2), ("slug", 3)], name="name", - indexes={"kind": "kind"}, + indexes={"slug": "slug"}, # unique: no two notes hold one slug; watch_by_index and get_key_by_index look it up + roles={"kind": "kind"}, # many notes share a kind: a selection alias, never a lookup ) node = CultMesh.start_node( "mesh.cc", @@ -55,13 +56,13 @@ node = CultMesh.start_node( node.database.register_document(note_doc) unsubscribe = node.database.watch_record(note_doc, "note:1", lambda change: print(change.change_kind)) unsubscribe_named = node.database.watch_by_name(note_doc, "intro", lambda change: print(change.record_key)) -unsubscribe_kind = node.database.watch_by_index(note_doc, "kind", "guide", lambda change: print(change.record_key)) -node.database.put(note_doc, "note:1", {"body": "hello", "name": "intro", "kind": "guide"}) +unsubscribe_slug = node.database.watch_by_index(note_doc, "slug", "intro-guide", lambda change: print(change.record_key)) +node.database.put(note_doc, "note:1", {"body": "hello", "name": "intro", "kind": "guide", "slug": "intro-guide"}) live_note = node.database.get_required(note_doc, "note:1") put_message = node.database.put_raw_message( note_doc, "note:2", - {"body": "wire me", "name": "wire", "kind": "interop"}, + {"body": "wire me", "name": "wire", "kind": "interop", "slug": "wire-me"}, shard_id="interop", shard_epoch=1, ) diff --git a/packages/cultmesh-py/src/cultmesh_py/node.py b/packages/cultmesh-py/src/cultmesh_py/node.py index cee102f0..619aed36 100644 --- a/packages/cultmesh-py/src/cultmesh_py/node.py +++ b/packages/cultmesh-py/src/cultmesh_py/node.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any, Callable from cultcache_py import CultCache, CultCacheEnvelope, SingleFileMessagePackBackingStore -from cultcache_py.cache import CultCacheError +from cultcache_py.cache import CultCacheError, index_text from cultcache_py.documents import DocumentDefinition, extract_value from cultnet_py import ( CultNetAppliedRecord, @@ -520,7 +520,7 @@ class CultMeshDatabase: raise ValueError(f"Document type {document.type!r} has no index {index!r}") def filtered(change: CultMeshDatabaseChange) -> None: - if self._change_matches_extractor(change, extractor, value): + if self._change_matches_extractor(change, extractor, value, index_text): callback(self._change_as_document(change, requested)) return self.watch(filtered, document=document) @@ -1295,10 +1295,11 @@ class CultMeshDatabase: change: CultMeshDatabaseChange, extractor: str | Callable[[Any], str | int | float | bool | None], expected: str, + render: Callable[[Any], str | None] = str, ) -> bool: return ( - self._value_matches_extractor(change.value, extractor, expected) - or self._value_matches_extractor(change.previous_value, extractor, expected) + self._value_matches_extractor(change.value, extractor, expected, render) + or self._value_matches_extractor(change.previous_value, extractor, expected, render) ) def _value_matches_extractor( @@ -1306,11 +1307,12 @@ class CultMeshDatabase: value: Any | None, extractor: str | Callable[[Any], str | int | float | bool | None], expected: str, + render: Callable[[Any], str | None] = str, ) -> bool: if value is None: return False actual = extract_value(value, extractor) - return actual is not None and str(actual) == expected + return actual is not None and render(actual) == expected def _previous_values_for_snapshot_response(self, response: dict[str, Any]) -> dict[tuple[str, str], Any]: if response.get("schemaVersion") != "cultnet.snapshot_response_raw.v0": diff --git a/packages/cultmesh-py/tests/test_unique_index_watch.py b/packages/cultmesh-py/tests/test_unique_index_watch.py new file mode 100644 index 00000000..ccc0b44d --- /dev/null +++ b/packages/cultmesh-py/tests/test_unique_index_watch.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +from cultcache_py import define_database_entry_type +from cultcache_py.cache import CultCacheError +from cultmesh_py import CultMesh + +# variants:ruling:unique-indexes. `indexes` are unique lookups, `roles` are selection aliases. A watch by index reads the +# lookup's own canonical text (cultcache_py.cache.index_text), never a second rendering of the value. + + +class WatchByIndexTests(unittest.TestCase): + def test_the_watch_and_the_lookup_agree_on_the_text_of_a_value(self) -> None: + document = define_database_entry_type("mesh.priced", [("price", 0)], indexes={"price": "price"}) + node = CultMesh.create_node(runtime_id="mesh-watch-text") + node.database.register_document(document) + cases = [ + (2.0, "2", "2.0"), + (0.1, "0.1000000000000000055511151231257827021181583404541015625", "0.1"), + (-0.0, "0", "-0.0"), + ] + for index, (value, text, str_text) in enumerate(cases): + with self.subTest(value=value): + key = f"item:{index}" + heard: list[str] = [] + misheard: list[str] = [] + node.database.watch_by_index(document, "price", text, lambda change: heard.append(change.record_key)) + node.database.watch_by_index(document, "price", str_text, lambda change: misheard.append(change.record_key)) + node.database.put(document, key, {"price": value}) + self.assertEqual(node.database.get_by_index(document, "price", text), {"price": value}) + self.assertEqual(heard, [key]) + if str_text != text: + self.assertEqual(misheard, []) + self.assertIsNone(node.database.get_by_index(document, "price", str_text)) + + def test_a_watch_hears_a_record_leaving_a_value_by_the_canonical_text(self) -> None: + document = define_database_entry_type("mesh.leaving", [("price", 0)], indexes={"price": "price"}) + node = CultMesh.create_node(runtime_id="mesh-watch-leaving") + node.database.register_document(document) + node.database.put(document, "item:1", {"price": 2.0}) + heard: list[str] = [] + node.database.watch_by_index(document, "price", "2", lambda change: heard.append(change.record_key)) + node.database.put(document, "item:1", {"price": 3.0}) + self.assertEqual(heard, ["item:1"]) + + def test_a_blank_value_is_never_indexed_so_never_watched(self) -> None: + document = define_database_entry_type("mesh.blank", [("sku", 0)], indexes={"sku": "sku"}) + node = CultMesh.create_node(runtime_id="mesh-watch-blank") + node.database.register_document(document) + heard: list[str] = [] + node.database.watch_by_index(document, "sku", " ", lambda change: heard.append(change.record_key)) + node.database.put(document, "item:1", {"sku": " "}) + self.assertEqual(heard, []) + + def test_a_role_alias_is_not_watchable_by_index(self) -> None: + document = define_database_entry_type("mesh.roles", [("kind", 0)], roles={"kind": "kind"}) + node = CultMesh.create_node(runtime_id="mesh-watch-role") + node.database.register_document(document) + with self.assertRaisesRegex(ValueError, "has no index"): + node.database.watch_by_index(document, "kind", "guide", lambda _: None) + + +class ReadmeTests(unittest.TestCase): + def test_the_readme_note_document_admits_many_notes_of_one_kind(self) -> None: + readme = (Path(__file__).parents[1] / "README.md").read_text(encoding="utf-8") + declaration = re.search(r"^note_doc = define_database_entry_type\(\n.*?^\)\n", readme, re.S | re.M) + self.assertIsNotNone(declaration) + scope: dict = {"define_database_entry_type": define_database_entry_type} + exec(declaration.group(0), scope) # noqa: S102 - the README's own declaration + note_doc = scope["note_doc"] + node = CultMesh.create_node(runtime_id="mesh-readme") + node.database.register_document(note_doc) + node.database.put(note_doc, "note:1", {"body": "a", "name": "one", "kind": "guide", "slug": "one"}) + node.database.put(note_doc, "note:2", {"body": "b", "name": "two", "kind": "guide", "slug": "two"}) + self.assertEqual(node.database.get_by_index(note_doc, "slug", "two")["body"], "b") + with self.assertRaises(CultCacheError): + node.database.put(note_doc, "note:3", {"body": "c", "name": "three", "kind": "guide", "slug": "one"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/CHANGELOG.md b/packages/cultnet-py/CHANGELOG.md index 5e4b0cfa..e7bd3afd 100644 --- a/packages/cultnet-py/CHANGELOG.md +++ b/packages/cultnet-py/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (breaking; version 0.x, so the next minor) +- `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move. - Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client. - Added `message_invalid` to `CULTNET_ERROR_CODES` and `message_invalid_error(field, reason)`, byte-identical to the C# `ForMessageInvalid`. `cultmesh-py` refuses a malformed put or delete with it instead of the out-of-schema `malformed_document_put` and `malformed_document_delete`, which are removed. - Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per diff --git a/packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py b/packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py index 91d0805b..bc5c2622 100644 --- a/packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py +++ b/packages/cultnet-py/src/cultnet_py/cultmesh_contracts.py @@ -692,7 +692,7 @@ simulation_fact_document: DocumentDefinition[CultMeshSimulationFact] = define_da ], cls=CultMeshSimulationFact, name="fact_id", - indexes={"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"}, + roles={"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"}, schema_name=SIMULATION_FACT_DOCUMENT_TYPE, schema_version=SIMULATION_FACT_SCHEMA_VERSION, ) diff --git a/packages/cultnet-py/src/cultnet_py/schema_catalog.py b/packages/cultnet-py/src/cultnet_py/schema_catalog.py index ee6fe430..b640b753 100644 --- a/packages/cultnet-py/src/cultnet_py/schema_catalog.py +++ b/packages/cultnet-py/src/cultnet_py/schema_catalog.py @@ -429,9 +429,11 @@ def wire_message_schema_properties(schema_version: str) -> dict[str, Any]: "messageId": common["messageId"], "subscriptionId": common["subscriptionId"], "changeKind": {"enum": ["added", "updated", "removed"]}, - "document": {"$ref": "#/$defs/rawDocumentRecord"}, - "schemaId": common["schemaId"], - "recordKey": common["recordKey"], + "document": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/rawDocumentRecord"}]}, + "schemaId": {"type": ["string", "null"], "minLength": 1}, + "recordKey": {"type": ["string", "null"], "minLength": 1}, + "commitId": {"type": ["string", "null"], "minLength": 1}, + "commitCount": {"type": ["integer", "null"], "minimum": 2}, }, "cultnet.shard_catalog_request.v0": { **common, diff --git a/packages/cultnet-py/src/cultnet_py/subscription.py b/packages/cultnet-py/src/cultnet_py/subscription.py index 6756376b..74c3be33 100644 --- a/packages/cultnet-py/src/cultnet_py/subscription.py +++ b/packages/cultnet-py/src/cultnet_py/subscription.py @@ -15,6 +15,10 @@ class CultNetDatabaseChange: raw_document: CultNetRawDocumentRecord | None = None schema_id: str | None = None record_key: str | None = None + commit_id: str | None = None + """Set on every change of one multi-record commit, the same for all of them; None outside one (C# writes nil).""" + commit_count: int | None = None + """How many changes the commit holds (>= 2); None outside one. Carried data: Python applies no batch.""" @classmethod def from_wire(cls, message: dict[str, Any]) -> "CultNetDatabaseChange": @@ -31,6 +35,12 @@ class CultNetDatabaseChange: if raw_document is not None: schema_id = schema_id or raw_document.schema_id record_key = record_key or raw_document.record_key + commit_id = message.get("commitId") + if commit_id is not None and not (isinstance(commit_id, str) and commit_id): + raise ValueError(f"database change commitId must be a non-empty string, received {commit_id!r}") + commit_count = message.get("commitCount") + if commit_count is not None and (isinstance(commit_count, bool) or not isinstance(commit_count, int) or commit_count < 2): + raise ValueError(f"database change commitCount must be an integer of at least 2, received {commit_count!r}") return cls( message_id=str(message.get("messageId") or ""), subscription_id=str(message.get("subscriptionId") or ""), @@ -39,6 +49,8 @@ class CultNetDatabaseChange: raw_document=raw_document, schema_id=schema_id, record_key=record_key, + commit_id=commit_id, + commit_count=commit_count, ) def to_wire(self) -> dict[str, Any]: @@ -56,6 +68,10 @@ class CultNetDatabaseChange: wire["schemaId"] = self.schema_id if self.record_key is not None: wire["recordKey"] = self.record_key + if self.commit_id is not None: + wire["commitId"] = self.commit_id + if self.commit_count is not None: + wire["commitCount"] = self.commit_count return wire diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 6e8d2b71..68f8f8ea 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -13,6 +13,7 @@ from datetime import UTC, datetime from pathlib import Path from typing import Callable from uuid import uuid4 +import msgpack from cultcache_py import CultCache, define_database_entry_type from cultnet_py.benchmark import run_benchmark from cultnet_py.compare_csharp import ( @@ -1566,6 +1567,51 @@ class CultNetTests(unittest.TestCase): self.assertEqual(delete_change.schema_id, "schema-note") self.assertEqual(delete_change.to_wire()["recordKey"], "note:1") + def test_cultnet_database_change_commit_fields(self) -> None: + base = { + "schemaVersion": "cultnet.database_change_raw.v0", + "messageId": "change-commit", + "subscriptionId": "sub-1", + "changeKind": "removed", + "schemaId": "schema-note", + "recordKey": "note:1", + } + grouped = CultNetDatabaseChange.from_wire({**base, "commitId": "commit-7", "commitCount": 3}) + self.assertEqual((grouped.commit_id, grouped.commit_count), ("commit-7", 3)) + self.assertEqual(grouped.to_wire()["commitId"], "commit-7") + self.assertEqual(grouped.to_wire()["commitCount"], 3) + self.assertEqual(CultNetDatabaseChange.from_wire(grouped.to_wire()), grouped) + + for absent in (base, {**base, "commitId": None, "commitCount": None}): # C# writes nil outside a commit + change = CultNetDatabaseChange.from_wire(absent) + self.assertIsNone(change.commit_id) + self.assertIsNone(change.commit_count) + self.assertNotIn("commitId", change.to_wire()) + self.assertNotIn("commitCount", change.to_wire()) + + for field, bad in ( + ("commitId", ""), + ("commitId", 7), + ("commitCount", 1), + ("commitCount", 0), + ("commitCount", True), + ("commitCount", 2.0), + ("commitCount", "3"), + ): + with self.subTest(field=field, bad=bad), self.assertRaisesRegex(ValueError, field): + CultNetDatabaseChange.from_wire({**base, field: bad}) + + def test_a_two_change_commit_written_by_csharp_parses(self) -> None: + # tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, written by the + # C# serializer (CultNetReplicaCommitTests), so a rename or retyping of commitId/commitCount fails here. + vector = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "database-change-commit" / "two-change-commit.msgpack" + messages = list(msgpack.Unpacker(io.BytesIO(vector.read_bytes()), raw=False)) + self.assertEqual(len(messages), 2) + changes = [CultNetDatabaseChange.from_wire(message) for message in messages] + self.assertEqual([change.commit_count for change in changes], [2, 2]) + self.assertEqual({change.commit_id for change in changes}, {"9f1c0f0e4b7a4c1f9d2e"}) + self.assertEqual([change.to_wire()["commitCount"] for change in changes], [2, 2]) + def test_cultnet_raw_put_helper_carries_message_id(self) -> None: put = document_put_raw( message_id="put-1", @@ -2269,9 +2315,12 @@ class CultNetTests(unittest.TestCase): change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"]) self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"]) - self.assertIn("document", change_schema["properties"]) - self.assertIn("schemaId", change_schema["properties"]) - self.assertIn("recordKey", change_schema["properties"]) + # The C# writer sets every key, nil where unset (a removed change's document, an added change's recordKey and schemaId). + self.assertEqual(change_schema["properties"]["document"]["oneOf"][0], {"type": "null"}) + self.assertEqual(change_schema["properties"]["schemaId"], {"type": ["string", "null"], "minLength": 1}) + self.assertEqual(change_schema["properties"]["recordKey"], {"type": ["string", "null"], "minLength": 1}) + self.assertEqual(change_schema["properties"]["commitId"], {"type": ["string", "null"], "minLength": 1}) + self.assertEqual(change_schema["properties"]["commitCount"], {"type": ["integer", "null"], "minimum": 2}) log_schema = json.loads(by_version["cultnet.shard_log_response.v0"]["schemaJson"]) self.assertIn("entries", log_schema["required"]) diff --git a/packages/cultnet-rs/tests/database_change_commit_vector.rs b/packages/cultnet-rs/tests/database_change_commit_vector.rs new file mode 100644 index 00000000..322227dc --- /dev/null +++ b/packages/cultnet-rs/tests/database_change_commit_vector.rs @@ -0,0 +1,43 @@ +//! `tests/vectors/database-change-commit/two-change-commit.msgpack` is written only by the C# reference: two +//! `cultnet.database_change_raw.v0` messages of one commit, every key present, nil where unset. cultnet-rs has no typed +//! `database_change_raw` message yet, so this walks the stream with the same `rmpv` values its wire decoder builds on and +//! pins the keys and values a Rust adopter will read. A rename of `commitId` or `commitCount` in the C# writer fails here. + +use rmpv::Value; +use std::io::Cursor; + +fn messages() -> Vec { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../tests/vectors/database-change-commit/two-change-commit.msgpack"); + let bytes = std::fs::read(path).expect("the vector is committed"); + let mut cursor = Cursor::new(bytes.as_slice()); + let mut out = Vec::new(); + while (cursor.position() as usize) < bytes.len() { + out.push(rmpv::decode::read_value(&mut cursor).expect("a msgpack value")); + } + out +} + +fn field<'a>(message: &'a Value, key: &str) -> &'a Value { + message + .as_map() + .and_then(|entries| entries.iter().find(|(k, _)| k.as_str() == Some(key))) + .map(|(_, v)| v) + .unwrap_or_else(|| panic!("the C# writer wrote no key {key}")) +} + +#[test] +fn a_two_change_commit_written_by_csharp_carries_its_commit_id_count_and_changes() { + let messages = messages(); + assert_eq!(messages.len(), 2); + for message in &messages { + assert_eq!(field(message, "schemaVersion").as_str(), Some("cultnet.database_change_raw.v0")); + assert_eq!(field(message, "commitId").as_str(), Some("9f1c0f0e4b7a4c1f9d2e")); + assert_eq!(field(message, "commitCount").as_u64(), Some(2)); + assert_eq!(field(message, "changeKind").as_str(), Some("removed")); + assert_eq!(field(message, "subscriptionId").as_str(), Some("sub-1")); + assert_eq!(field(message, "schemaId").as_str(), Some("schema-note")); + assert!(field(message, "document").is_nil(), "a removed change's document is nil"); + } + let keys: Vec<_> = messages.iter().map(|m| field(m, "recordKey").as_str().unwrap().to_string()).collect(); + assert_eq!(keys, ["note:a", "note:b"]); +} diff --git a/packages/cultnet-ts/README.md b/packages/cultnet-ts/README.md index 07996956..7d1b9a6a 100644 --- a/packages/cultnet-ts/README.md +++ b/packages/cultnet-ts/README.md @@ -66,6 +66,7 @@ Current message families: - `cultnet.snapshot_response.v0` - `cultnet.snapshot_response_raw.v0` - `cultnet.snapshot_request.v1`, `cultnet.database_subscribe.v1`, `cultnet.snapshot_response_raw.v1` (typed-selection carriers; no selection engine yet) +- `cultnet.database_change_raw.v0` (optional `commitId` and `commitCount >= 2` mark the changes of one commit; carried and validated here, no TS code groups or applies by them) - `cultnet.sample.change_name.v0` - `cultnet.sample.chat.v0` diff --git a/packages/cultnet-ts/src/contracts.ts b/packages/cultnet-ts/src/contracts.ts index 8dd37ced..b733c959 100644 --- a/packages/cultnet-ts/src/contracts.ts +++ b/packages/cultnet-ts/src/contracts.ts @@ -424,6 +424,10 @@ export interface CultNetDatabaseChangeRawMessage { document?: CultNetRawDocumentRecord; recordKey?: string; schemaId?: string; + /** Opaque, one subscription: changes sharing a commitId are the changes of one commit. */ + commitId?: string | null; + /** How many changes the commit carries, at least two. The C# server sends null on a change outside a grouped commit. */ + commitCount?: number | null; } export interface CultNetSchemaDescriptor { @@ -1397,6 +1401,15 @@ function validateDatabaseChangeRawMessage( ); } + if (candidate.commitId !== undefined && candidate.commitId !== null && + (typeof candidate.commitId !== "string" || candidate.commitId.length === 0)) { + throw new Error("Validation failed for cultnet.database_change_raw.v0: /commitId: must be a non-empty string"); + } + if (candidate.commitCount !== undefined && candidate.commitCount !== null && + (typeof candidate.commitCount !== "number" || !Number.isInteger(candidate.commitCount) || candidate.commitCount < 2)) { + throw new Error("Validation failed for cultnet.database_change_raw.v0: /commitCount: must be an integer of at least 2"); + } + if (candidate.changeKind === "removed") { requireNonEmptyString(candidate.recordKey, "/recordKey"); requireNonEmptyString(candidate.schemaId, "/schemaId"); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 37d963a8..ac4f539d 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -54,6 +54,7 @@ import { validateGhostlightAgentStateGenerated, validateGhostlightAgentState, shardServes, + type CultNetDatabaseChangeRawMessage, type CultNetLoginMessage, type GhostlightAgentStateShape, type GhostlightAgentStateDocument, @@ -1688,6 +1689,55 @@ test("database subscription contracts match the C# live document lane", () => { assert.equal(removed.schemaVersion, "cultnet.database_change_raw.v0"); assert.equal(removed.recordKey, "counter:main"); + const grouped = parseCultNetMessage({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: "change-counter-3", + subscriptionId: "counter:browser-1", + changeKind: "removed", + recordKey: "counter:main", + schemaId: "sample.counter_state.v1", + commitId: "c1", + commitCount: 2, + }); + assert.equal((grouped as CultNetDatabaseChangeRawMessage).commitId, "c1"); + assert.equal((grouped as CultNetDatabaseChangeRawMessage).commitCount, 2); + assert.deepEqual(encodeCultNetMessageForWire(grouped), grouped, "both commit fields travel"); + assert.equal((removed as CultNetDatabaseChangeRawMessage).commitId, undefined, "a change without them parses as before"); + const changeWith = (extra: Record) => () => parseCultNetMessage({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: "change-counter-4", + subscriptionId: "counter:browser-1", + changeKind: "removed", + recordKey: "counter:main", + schemaId: "sample.counter_state.v1", + ...extra, + }); + // The C# server writes commitId and commitCount as nil on every change outside a grouped commit, so null is absent. + const ungrouped = changeWith({ commitId: null, commitCount: null })() as CultNetDatabaseChangeRawMessage; + assert.equal(ungrouped.commitId, null); + assert.equal(ungrouped.commitCount, null); + assert.doesNotThrow(changeWith({ commitId: null })); + assert.doesNotThrow(changeWith({ commitCount: null })); + assert.doesNotThrow(() => parseCultNetMessage({ + schemaVersion: "cultnet.database_change_raw.v0", + messageId: "change-counter-5", + subscriptionId: "counter:browser-1", + changeKind: "added", + document: { + schemaId: "sample.counter_state.v1", + recordKey: "counter:main", + storedAt: "2026-08-17T00:00:00Z", + payloadEncoding: "messagepack", + payload: new Uint8Array([0x81, 0xa5, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x01]), + }, + recordKey: null, + schemaId: null, + commitId: null, + commitCount: null, + })); + for (const commitId of ["", 5]) assert.throws(changeWith({ commitId }), /\/commitId/, `commitId ${JSON.stringify(commitId)}`); + for (const commitCount of [1, 0, 2.5, "two", -3]) assert.throws(changeWith({ commitCount }), /\/commitCount/, `commitCount ${commitCount}`); + const unsubscribe = parseCultNetMessage({ schemaVersion: "cultnet.database_unsubscribe.v0", messageId: "unsubscribe-counter", diff --git a/packages/cultnet-ts/test/database-change-commit-vector.test.ts b/packages/cultnet-ts/test/database-change-commit-vector.test.ts new file mode 100644 index 00000000..fea121c0 --- /dev/null +++ b/packages/cultnet-ts/test/database-change-commit-vector.test.ts @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import Ajv2020 from "ajv/dist/2020.js"; +import { decodeMulti } from "@msgpack/msgpack"; + +import { cultNetSchemas, parseCultNetMessage } from "../src/index"; + +// tests/vectors/database-change-commit/two-change-commit.msgpack is written only by the C# reference: two removed +// database_change_raw messages of one commit, every key present, nil where unset (document included). +const vector = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "database-change-commit", "two-change-commit.msgpack"); +const changes = () => [...decodeMulti(readFileSync(vector))] as Record[]; + +test("a two-change commit written by C# decodes with its commitId, commitCount and changes", () => { + const messages = changes().map((message) => parseCultNetMessage(structuredClone(message))); + assert.equal(messages.length, 2); + for (const message of messages) { + assert.equal(message.schemaVersion, "cultnet.database_change_raw.v0"); + assert.equal((message as { commitId?: unknown }).commitId, "9f1c0f0e4b7a4c1f9d2e"); + assert.equal((message as { commitCount?: unknown }).commitCount, 2); + assert.equal((message as { changeKind?: unknown }).changeKind, "removed"); + assert.equal((message as { subscriptionId?: unknown }).subscriptionId, "sub-1"); + assert.equal((message as { schemaId?: unknown }).schemaId, "schema-note"); + } + assert.deepEqual(messages.map((message) => (message as { recordKey?: unknown }).recordKey), ["note:a", "note:b"]); +}); + +test("the database_change_raw schema accepts the C# vector's removed changes (document: nil) and still refuses an added change without a document", () => { + const ajv = new Ajv2020({ allErrors: true, strict: false }); + ajv.addSchema(cultNetSchemas.rawDocumentRecordSchema); + const validate = ajv.compile(cultNetSchemas.databaseChangeRawSchema); + for (const change of changes()) { + assert.equal(change.document, null, "the C# writer wrote the key, nil"); + assert.equal(validate(change), true, JSON.stringify(validate.errors)); + assert.equal(validate({ ...change, changeKind: "added" }), false, "an added change needs a document"); + } +}); diff --git a/packages/cultnet-ts/test/governance-vectors.test.ts b/packages/cultnet-ts/test/governance-vectors.test.ts index a1f763fe..d77ca936 100644 --- a/packages/cultnet-ts/test/governance-vectors.test.ts +++ b/packages/cultnet-ts/test/governance-vectors.test.ts @@ -208,6 +208,7 @@ test("fork card: documents validate and encode to the vector bytes", () => { assert.ok(check(forkSchema, "refusal", v.message), v.name); assert.equal(hexOf(v.message), v.messagePackHex, v.name); } + assert.equal(fork.doorRefusals.find((r: { name: string }) => r.name === "fork_too_large").httpStatus, 422); }); test("fork card: every negative is refused by the schema or a stated rule", () => { @@ -387,6 +388,17 @@ const entryRuleErrors = (e: { card: Card; record: { fork_id: string; options_has return errors; }; +// Order is a unique rank per Mind: the refusal names both fork ids, the first holder and the second. +const orderErrors = (cards: { fork_id: string; order: number }[]): string[] => { + const holder = new Map(); + const errors: string[] = []; + for (const c of cards) { + if (holder.has(c.order)) errors.push(`order ${c.order} is shared by ${holder.get(c.order)} and ${c.fork_id}`); + else holder.set(c.order, c.fork_id); + } + return errors; +}; + // The reference list judge: newest first by card order, status filter, cursor c = cards below order N, a page of limit, counts over the whole set. const judgeList = (entries: { card: Card }[], ordinal: number, rq: { mind: string; status: string | null; cursor: string | null; limit: number }) => { const rows = [...entries].sort((a, b) => (b.card as any).order - (a.card as any).order); @@ -461,7 +473,9 @@ test("fork transport: requests and entries validate and encode to the vector byt } } for (const v of t.listInvalid) { - assert.equal(check(forkSchema, "list", v.message), false, `${v.name}: ${v.why}`); + const refused = !check(forkSchema, "list", v.message) || orderErrors(v.message.forks.map((f: any) => f.card)).length > 0; + assert.ok(refused, `${v.name}: ${v.why}`); + if (v.refusal !== undefined) assert.deepEqual(orderErrors(v.message.forks.map((f: any) => f.card)), [v.refusal], v.name); assert.equal(hexOf(v.message), v.messagePackHex, v.name); } }); @@ -469,6 +483,7 @@ test("fork transport: requests and entries validate and encode to the vector byt test("fork transport: every list request replays to the vector's response, newest first, paged by the cursor", () => { for (const sc of fork.transport.listScenarios) { for (const e of sc.entries) assert.deepEqual(entryRuleErrors(e), [], sc.name); + assert.deepEqual(orderErrors(sc.entries.map((e: any) => e.card)), [], sc.name); let previous: { next_cursor: string | null } | null = null; for (const step of sc.steps) { const response = judgeList(sc.entries, sc.ordinal, step.request); @@ -483,6 +498,95 @@ test("fork transport: every list request replays to the vector's response, newes } }); +test("fork transport: order 0 is legal, no cursor names it, and the page ending at order 1 hands on to it", () => { + const sc = fork.transport.listScenarios.find((x: any) => x.name === "order_zero_is_legal_but_no_cursor_names_it"); + assert.deepEqual(sc.steps.map((s: any) => s.response.next_cursor), ["c1", null]); + assert.deepEqual(sc.steps[1].response.forks.map((f: any) => f.card.order), [0]); + assert.equal(check(forkSchema, "cursor", "c0"), false); +}); + +test("fork transport: a page may stop short of limit, and a first entry past the budget is fork_too_large (422) continued below its order", () => { + const sc = fork.transport.listShortPages; + assert.deepEqual(sc.map((x: any) => x.name), ["a_short_page_resumes_below_its_last_entry", "a_first_entry_past_the_budget_is_fork_too_large_and_the_client_continues_below_its_order", "fork_too_large_on_order_zero_leaves_nothing_below"]); + assert.ok((forkSchema.$defs.refusal.properties.code as { enum: string[] }).enum.includes("fork_too_large")); + const covered: number[] = []; + for (const s of sc) { + let last: string | null | undefined = undefined; + s.steps.forEach((step: any, i: number) => { + if (last !== undefined) assert.equal(step.request.cursor, last, `${s.name}: the step follows the cursor or the refused order`); + const full = judgeList(s.entries, s.ordinal, step.request); + if (step.response !== undefined) { + const r = step.response as any, n = r.forks.length; + assert.ok(check(forkSchema, "list", r), s.name); + assert.equal(hexOf(r), step.responseHex, s.name); + assert.deepEqual(r.forks, full.forks.slice(0, n), `${s.name}: a prefix of the unbudgeted page`); + assert.ok(n >= 1, s.name); + assert.equal(r.next_cursor, full.next_cursor !== null || n < full.forks.length ? `c${r.forks[n - 1].card.order}` : null, s.name); + assert.equal(r.stamp, full.stamp); assert.equal(r.open_count, full.open_count); assert.equal(r.recorded_count, full.recorded_count); + covered.push(...r.forks.map((f: any) => f.card.order)); + last = r.next_cursor; + } else { + const r = step.refusal, first = full.forks[0].card as any; + assert.ok(check(forkSchema, "refusal", r), s.name); + assert.equal(hexOf(r), step.refusalHex, s.name); + assert.equal(r.code, "fork_too_large"); + assert.equal(step.httpStatus, 422); + assert.equal(step.fork_id, first.fork_id); assert.equal(step.order, first.order); + assert.ok(r.error.includes(first.fork_id) && new RegExp(`\\b${first.order}\\b`).test(r.error.replace(first.fork_id, "")), `${s.name}: the text names the fork id and the order`); + assert.ok(step.request.limit >= 1, "limit 0 never refuses"); + if (first.order >= 1) { assert.ok(i + 1 < s.steps.length, s.name); last = `c${first.order}`; assert.ok(check(forkSchema, "cursor", last)); } + else { assert.equal(i, s.steps.length - 1, `${s.name}: order 0 is the end`); assert.equal(check(forkSchema, "cursor", "c0"), false); last = undefined; } + } + }); + } + assert.deepEqual(sc[0].steps.map((x: any) => x.response.next_cursor), ["c2", null]); + assert.deepEqual(sc[0].steps.map((x: any) => x.response.forks.length), [2, 2]); + assert.deepEqual([sc[1].steps[0].order, sc[2].steps[1].order], [3, 0]); + assert.deepEqual(covered, [3, 2, 1, 0, 2, 1, 0, 1]); +}); + +test("fork transport: a read whose entry is past the cap is fork_too_large (422), named by fork id alone", () => { + const doorStatus = fork.doorRefusals.find((r: any) => r.name === "fork_too_large").httpStatus; + assert.equal(fork.transport.readTooLarge.length, 1); + for (const c of fork.transport.readTooLarge) { + const held = c.entries.find((e: any) => e.card.fork_id === c.request.fork_id); + assert.ok(held !== undefined && check(forkSchema, "readRequest", c.request), c.name); + assert.ok(check(forkSchema, "refusal", c.refusal), c.name); + assert.equal(c.refusal.code, "fork_too_large"); + assert.ok(c.refusal.error.includes(held.card.fork_id), "the text names the fork id"); + assert.equal(hexOf(c.refusal), c.refusalHex, c.name); + assert.equal(c.httpStatus, 422); assert.equal(c.httpStatus, doorStatus); + } +}); + +test("fork transport: a stale answer whose card re-read is past the cap is the read-form fork_too_large (422), over a stored Refused(stale_fork)", () => { + assert.equal(fork.transport.answerTooLarge.length, 1); + const read = fork.transport.readTooLarge[0]; + for (const c of fork.transport.answerTooLarge) { + const held = c.entries.find((e: any) => e.card.fork_id === c.request.fork_id); + assert.ok(held !== undefined && check(forkSchema, "answerRequest", c.request), c.name); + assert.notEqual(c.request.options_hash, held.card.options_hash, "the card moved: the answer is stale"); + assert.ok(check(forkSchema, "refusal", c.refusal), c.name); + assert.equal(c.refusal.code, "fork_too_large"); + assert.ok(c.refusal.error.includes(held.card.fork_id), "the text names the fork id"); + assert.equal(hexOf(c.refusal), c.refusalHex, c.name); + assert.equal(c.refusal.code, read.refusal.code, "the same refusal as the read form"); + assert.equal(c.httpStatus, 422); assert.equal(c.httpStatus, fork.doorRefusals.find((r: any) => r.name === "fork_too_large").httpStatus); + assert.equal(c.outcome, "Refused"); assert.equal(c.code, "stale_fork"); assert.equal(c.counted, false); + } +}); + +test("fork transport: a door request_id_conflict is a fault on read and list (503 unavailable) and a refusal only on the answer", () => { + assert.deepEqual(fork.transport.doorFaults.map((c: any) => c.route), ["read", "list", "answer"]); + for (const c of fork.transport.doorFaults) { + const source = c.route === "answer" ? fork.doorRefusals.find((r: any) => r.name === c.doorCode) : fork.unavailable[0]; + assert.equal(c.responseHex, source.messagePackHex, c.name); + assert.equal(hexOf(c.response), c.responseHex, c.name); + assert.equal(c.httpStatus, source.httpStatus, c.name); + assert.ok(check(forkSchema, c.route === "answer" ? "refusal" : "unavailable", c.response), c.name); + } +}); + test("fork transport: auth is bearer, then audience, then scope, and a votes token touches nothing", () => { const codes = (forkSchema.$defs.refusal.properties.code as { enum: string[] }).enum; const seen = new Set(); diff --git a/scripts/gen-governance-vectors.py b/scripts/gen-governance-vectors.py index de42df47..ec2a269a 100644 --- a/scripts/gen-governance-vectors.py +++ b/scripts/gen-governance-vectors.py @@ -358,7 +358,7 @@ scen += [ sc("R1_a_target_less_retraction_with_retracted_flipped_is_body_invalid_before_any_request_id_lookup", CARD, [X1, rf(IL_B, sr("n2"), "bad_retraction"), door(dict(IL_B, retracted=False), sr("n9"), "body_invalid")], "n1"), ] REASON_CODES = ["stale_fork", "not_a_decider", "fork_closed", "choice_invalid", "bad_retraction", "already_retracted"] -HTTP = {"unauthenticated": 401, "body_forbidden_field": 400, "body_unknown_field": 400, "body_invalid": 400, "fork_unknown": 404, "insufficient_scope": 403, "forbidden": 403, "request_id_conflict": 409, "request_invalid": 400, +HTTP = {"unauthenticated": 401, "body_forbidden_field": 400, "body_unknown_field": 400, "body_invalid": 400, "fork_unknown": 404, "insufficient_scope": 403, "forbidden": 403, "request_id_conflict": 409, "request_invalid": 400, "fork_too_large": 422, "stale_fork": 409, "not_a_decider": 403, "fork_closed": 409, "choice_invalid": 400, "bad_retraction": 400, "already_retracted": 409} STORED_REFUSALS = [] for reason in REASON_CODES: @@ -393,6 +393,16 @@ a5 = da("Counted", None, None, fbody(q5, "no", NOTE), srv("n1", OP, T(2))) a3 = da("Counted", None, None, fbody(q3, "yes", NOTE), srv("n1", OP, T(2))) r3 = da("Refused", None, "fork_closed", fbody(q3, "no", NOTE), srv("n2", HM, T(4), "Steward")) ENTRIES = [fentry(q5, [a5], "n1"), fentry(q4), fentry(q3, [a3, r3], "n1"), fentry(q2), fentry(q1)] +q0 = fcard(0) +ORDER0 = [fentry(fcard(3)), fentry(fcard(2)), fentry(fcard(1)), fentry(q0)] # a Mind whose oldest card has order 0 +q4b = dict(fcard(4), fork_id="gamecult-chat:question:q4b") # a second card on order 4 in the same Mind +def order_shared(entries): + seen, out = {}, [] + for e in entries: + c = e["card"] + if c["order"] in seen: out.append(f"order {c['order']} is shared by {seen[c['order']]} and {c['fork_id']}") + else: seen[c["order"]] = c["fork_id"] + return out def lreq(mind="gamecult-chat", status=None, cursor=None, limit=10): return {"schemaVersion": "gamecult.fork.list_request.v0", "mind": mind, "status": status, "cursor": cursor, "limit": limit} def judge_list(entries, ordinal, rq): rows = sorted(entries, key=lambda e: -e["card"]["order"]) @@ -409,8 +419,38 @@ LISTS = [ {"name": "all_forks_one_page", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(limit=50))]}, {"name": "recorded_only_paged", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(status="recorded", limit=1)), lstep(ENTRIES, 412, lreq(status="recorded", cursor="c3", limit=1))]}, {"name": "limit_zero_is_the_count_and_stamp_only", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(limit=0))]}, + {"name": "order_zero_is_legal_but_no_cursor_names_it", "ordinal": 9, "entries": ORDER0, "steps": [lstep(ORDER0, 9, lreq(limit=3)), lstep(ORDER0, 9, lreq(cursor="c1", limit=3))]}, {"name": "a_mind_with_no_forks", "ordinal": 7, "entries": [], "steps": [lstep([], 7, lreq(mind="nibu", limit=10))]}, ] +# A page may stop short of `limit` when the next entry would pass the receiver's response budget; a list whose first +# entry alone passes it is refused `fork_too_large` (422). The budget is not in the vector: a step records what the receiver answered, and the +# judges check it against the unbudgeted page (a nonempty strict prefix, resumed below its last entry) or against the entry the page would open with. +def too_large_doc(c): return {"schemaVersion": "gamecult.fork.refusal.v0", "code": "fork_too_large", "error": f"fork_too_large: fork {c['fork_id']} (order {c['order']}) alone is past the response budget"} +def read_too_large_doc(c): return {"schemaVersion": "gamecult.fork.refusal.v0", "code": "fork_too_large", "error": f"fork_too_large: fork {c['fork_id']} alone is past the response budget"} +READ_TOO_LARGE = [{"name": "a_read_whose_entry_is_past_the_cap_is_fork_too_large_without_an_order", "ordinal": 9, "entries": ORDER0, + "request": {"schemaVersion": "gamecult.fork.read_request.v0", "mind": "gamecult-chat", "fork_id": ORDER0[0]["card"]["fork_id"]}, + "refusal": read_too_large_doc(ORDER0[0]["card"]), "refusalHex": hx(read_too_large_doc(ORDER0[0]["card"])), "httpStatus": HTTP["fork_too_large"], "fork_id": ORDER0[0]["card"]["fork_id"]}] +# An answer on a stale card whose current-card re-read is past the cap: the same refusal document as the read form over a stored Refused(stale_fork), never counted. +_big = ORDER0[0]["card"] +ANSWER_TOO_LARGE = [{"name": "a_stale_answer_whose_card_reread_is_past_the_cap_is_fork_too_large_over_a_stored_stale_refusal", + "entries": ORDER0, "request": body("yes", NOTE, h=STALE_H, fork=_big["fork_id"], rid="0000a001-0000-4000-8000-00000000a001"), + "refusal": read_too_large_doc(_big), "refusalHex": hx(read_too_large_doc(_big)), "httpStatus": HTTP["fork_too_large"], "outcome": "Refused", "code": "stale_fork", "counted": False, "fork_id": _big["fork_id"]}] +def short_step(entries, ordinal, rq, keep): + full = judge_list(entries, ordinal, rq); page = full["forks"][:keep] + assert 0 < keep < len(full["forks"]) + r = dict(full, forks=page, next_cursor=f"c{page[-1]['card']['order']}"); return {"request": rq, "response": r, "responseHex": hx(r)} +def large_step(entries, ordinal, rq): + c = judge_list(entries, ordinal, rq)["forks"][0]["card"]; r = too_large_doc(c) + return {"request": rq, "refusal": r, "refusalHex": hx(r), "httpStatus": HTTP["fork_too_large"], "fork_id": c["fork_id"], "order": c["order"]} +SHORT_PAGES = [ + {"name": "a_short_page_resumes_below_its_last_entry", "ordinal": 9, "entries": ORDER0, "steps": [short_step(ORDER0, 9, lreq(limit=3), 2), lstep(ORDER0, 9, lreq(cursor="c2", limit=3))]}, + {"name": "a_first_entry_past_the_budget_is_fork_too_large_and_the_client_continues_below_its_order", "ordinal": 9, "entries": ORDER0, + "steps": [large_step(ORDER0, 9, lreq(limit=3)), lstep(ORDER0, 9, lreq(cursor="c3", limit=3))]}, + {"name": "fork_too_large_on_order_zero_leaves_nothing_below", "ordinal": 9, "entries": ORDER0[2:], + "steps": [short_step(ORDER0[2:], 9, lreq(limit=5), 1), large_step(ORDER0[2:], 9, lreq(cursor="c1", limit=5))]}, +] +DUP_LIST = judge_list(ENTRIES + [fentry(q4b)], 412, lreq(limit=50)) +assert order_shared(DUP_LIST["forks"]) == ["order 4 is shared by gamecult-chat:question:q4 and gamecult-chat:question:q4b"] TOK = lambda aud, *sc, active=True: {"audience": aud, "scopes": list(sc), "active": active} AUTH = [ ("list_with_desk_read", "list", TOK("bifrost-desk", "desk.read"), "ok"), ("read_with_desk_read", "read", TOK("bifrost-desk", "desk.read"), "ok"), @@ -573,7 +613,7 @@ fork = { ("no_answerer", "answerer is stamped on every stored answer", dict(CNT, answerer="")), ]]}, "results": [{"name": r["name"], "httpStatus": r["httpStatus"], "message": r["message"], "messagePackHex": hx(r["message"])} for r in STORED_REFUSALS], - "doorRefusals": [{"name": c, "httpStatus": HTTP[c], "message": refusal_doc(c), "messagePackHex": hx(refusal_doc(c))} for c in TAIL], + "doorRefusals": [{"name": c, "httpStatus": HTTP[c], "message": refusal_doc(c), "messagePackHex": hx(refusal_doc(c))} for c in TAIL] + [{"name": "fork_too_large", "httpStatus": HTTP["fork_too_large"], "message": SHORT_PAGES[1]["steps"][0]["refusal"], "messagePackHex": SHORT_PAGES[1]["steps"][0]["refusalHex"]}], "unavailable": [{"name": "second_door_down", "httpStatus": 503, "message": unavailable_doc(30), "messagePackHex": hx(unavailable_doc(30))}, {"name": "second_door_down_no_hint", "httpStatus": 503, "message": unavailable_doc(None), "messagePackHex": hx(unavailable_doc(None))}], "rulingCites": cites, @@ -601,8 +641,16 @@ fork = { "listInvalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [ ("next_cursor_malformed", "a cursor is c", dict(LISTS[0]["steps"][0]["response"], next_cursor="more")), ("stamp_malformed", "a stamp is s", dict(LISTS[0]["steps"][0]["response"], stamp="412")), - ("count_negative", "counts are not negative", dict(LISTS[0]["steps"][0]["response"], open_count=-1))]], + ("count_negative", "counts are not negative", dict(LISTS[0]["steps"][0]["response"], open_count=-1))]] + + [dict(bad("two_cards_share_one_order", "rule: order is a unique rank per Mind; the refusal names both fork ids (not expressible in the schema)", DUP_LIST), + messagePackHex=hx(DUP_LIST), refusal=order_shared(DUP_LIST["forks"])[0])], "listScenarios": [{"name": l["name"], "ordinal": l["ordinal"], "entries": l["entries"], "steps": l["steps"]} for l in LISTS], + "listShortPages": SHORT_PAGES, + "readTooLarge": READ_TOO_LARGE, + "answerTooLarge": ANSWER_TOO_LARGE, + "doorFaults": [dict(c, response=r, responseHex=hx(r)) for c, r in + [({"name": f"{x}_door_answers_request_id_conflict_the_client_sees_unavailable", "route": x, "doorCode": "request_id_conflict", "httpStatus": 503}, unavailable_doc(30)) for x in ("read", "list")] + + [({"name": "answer_door_answers_request_id_conflict_the_client_sees_it", "route": "answer", "doorCode": "request_id_conflict", "httpStatus": HTTP["request_id_conflict"]}, refusal_doc("request_id_conflict"))]], "mindRoutes": {"minds": MINDS, "forks": MIND_FORKS, "cases": MINDCASES, "sameResponse": SAME_RESPONSE}, "authCases": [{"name": a[0], "route": a[1], "token": a[2], "expect": a[3], "huginn_down": len(a) > 4} for a in AUTH], }, diff --git a/src/GameCult.Caching.MessagePack/README.md b/src/GameCult.Caching.MessagePack/README.md index d34490f7..5e7156b0 100644 --- a/src/GameCult.Caching.MessagePack/README.md +++ b/src/GameCult.Caching.MessagePack/README.md @@ -59,6 +59,10 @@ await cache.PullAllBackingStoresAsync(); - The raw CultNet document lane should treat these payload bytes as already blessed, not decode and re-encode them for sport. +## Store format and rollback + +A store whose elements hold an id is marked `cultcache.store.v3` (a directory store `v5`); the single-file and directory stores here write the mark from content. See the Store format and rollback section of the GameCult.Caching README. Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. The only carry-back is narrow and lossy (a whole-store write that leaves no id in the store drops the mark and the ids); otherwise restore from a copy taken before upgrading, and upgrade every reader before any writer. + ## MessagePack source generator NuGet consumers get `[assembly: MessagePackAssumedFormattable(typeof(CultRecordRef<>))]` from the package, so a diff --git a/src/GameCult.Caching/Contracts/cultcache-store-composition.md b/src/GameCult.Caching/Contracts/cultcache-store-composition.md index 1a03d4b1..01f7a5c9 100644 --- a/src/GameCult.Caching/Contracts/cultcache-store-composition.md +++ b/src/GameCult.Caching/Contracts/cultcache-store-composition.md @@ -110,7 +110,10 @@ which route wrote it. already agree when publication starts. - An admission runs a record's `[CultName]` and `[CultIndex]` getters once per landing record, before the store is written and before any `Sequence` is - minted. A conditional commit on a clean file store is judged a second time + minted. An admission is refused when it would leave two records of one type + holding one `[CultIndex]` value (blank is no value), and a load of a store + holding such a pair refuses naming both keys and the index. Concurrent writers on a + directory store are not covered by this rule. A conditional commit on a clean file store is judged a second time against the file it merges onto whenever the cache holds variants in any store, the batch lands a variant, or the file holds one. That second read runs the getters again for the records the batch lands, for any variant it re-resolves, diff --git a/src/GameCult.Caching/CultCache.cs b/src/GameCult.Caching/CultCache.cs index 5a0626a6..6f04d99c 100644 --- a/src/GameCult.Caching/CultCache.cs +++ b/src/GameCult.Caching/CultCache.cs @@ -885,6 +885,10 @@ namespace GameCult.Caching return indexes.BySchemaId.ContainsKey(schemaId) || indexes.ByCompatibleSchemaId.ContainsKey(schemaId); } + // Whether the type registered under this schema id declares a [CultIndex]: its records are written through a cache, which judges uniqueness. + internal bool HoldsUniqueIndex(string schemaId) => + _indexes.BySchemaId.TryGetValue(schemaId, out var descriptor) && descriptor.IndexAccessors.Count > 0; + // A persisted entry a local entry reads whole: identical content, or compatible shapes that leave no persisted slot ignored. internal static bool ReadsWhole(CultSchemaCatalogEntry persisted, CultSchemaCatalogEntry local) => string.Equals(persisted.ContentHash, local.ContentHash, StringComparison.Ordinal) || @@ -1006,10 +1010,10 @@ namespace GameCult.Caching throw new InvalidOperationException(ids.Problems[0]); var nameMember = members.FirstOrDefault(member => member.IsName); var indexAccessors = members - .Where(member => member.IndexAlias != null) + .Where(member => member.IsUnique) .ToDictionary( member => member.IndexAlias!, - member => member.Getter, + member => member.IndexValue, StringComparer.Ordinal); var descriptorMembers = members .Select(member => new CultDocumentMemberDescriptor @@ -1481,6 +1485,13 @@ namespace GameCult.Caching rejections.Add(HiddenMemberMessage(type.Name, Qualified(pair[0].Member), Qualified(pair[1].Member))); } + // A member names its role once: [CultIndex] already is a role. + foreach (var entry in keyed.OrderBy(entry => entry.Member.Name, StringComparer.Ordinal)) + { + if (entry.Member.GetCustomAttribute() != null && entry.Member.GetCustomAttribute() != null) + rejections.Add($"Cult document {type.Name} member {Qualified(entry.Member)} declares both [CultIndex] and [CultRole]; [CultIndex] already names its role."); + } + // D10: two differently-named members that independently declare the same index alias. A // hiding member sharing one slot is already caught above; this is two distinct members whose // alias map entry (BuildDescriptor's indexAccessors) would otherwise collide silently. @@ -1580,14 +1591,19 @@ namespace GameCult.Caching private static string Qualified(MemberInfo member) => member.DeclaringType!.Name.Split('`')[0] + "." + member.Name; + // The member's role alias, from [CultIndex] or [CultRole] (the member name when the alias is blank). // Shared by DiscoverMembers' D10/D11 rejections and PersistedMember.FromMember, so registration // refuses exactly the shapes the accessor it builds afterward could not have walked anyway. private static string? ResolveIndexAlias(MemberInfo member) { - var indexAttribute = member.GetCustomAttribute(); - if (indexAttribute == null) + string? alias; + if (member.GetCustomAttribute() is { } index) + alias = index.Alias; + else if (member.GetCustomAttribute() is { } role) + alias = role.Alias; + else return null; - return string.IsNullOrWhiteSpace(indexAttribute.Alias) ? member.Name : indexAttribute.Alias; + return string.IsNullOrWhiteSpace(alias) ? member.Name : alias; } private static bool IsCultRecordRefType(Type type) => @@ -1854,6 +1870,7 @@ namespace GameCult.Caching public int Slot { get; set; } public bool IsName { get; set; } public string? IndexAlias { get; set; } + public bool IsUnique { get; set; } public bool IsReference { get; set; } public bool IsMany { get; set; } public string? TargetSchemaName { get; set; } @@ -1866,6 +1883,10 @@ namespace GameCult.Caching public Func Getter { get; set; } = default!; public Func GetterNullable { get; set; } = default!; + // What a [CultIndex] value is: culture-invariant, and a number is its canonical decimal (1.0m and 1.00m are one value, + // 0.0 and -0.0 are one value), so the value that decides a refusal is the value a lookup names. + public Func IndexValue { get; set; } = default!; + // D8/Q-J (revised 2026-09-22): non-null exactly for the closed CLR numeric set (Nullable // unwrapped), rendered as the member's exact canonical decimal string - never a double, so // no CLR numeric type's range or precision is lost rendering it. docs/cultnet-selection-cut.md @@ -1896,6 +1917,7 @@ namespace GameCult.Caching Slot = slot, IsName = member.GetCustomAttribute() != null, IndexAlias = ResolveIndexAlias(member), + IsUnique = member.GetCustomAttribute() != null, IsReference = isReference, IsMany = many, TargetSchemaName = targetSchemaName, @@ -1903,6 +1925,13 @@ namespace GameCult.Caching IsNumeric = IsNumericClrType(memberType), Getter = document => getValue(document)?.ToString() ?? string.Empty, GetterNullable = document => getValue(document)?.ToString(), + IndexValue = document => getValue(document) switch + { + null => string.Empty, + var raw when IsNumericClrType(raw.GetType()) => RenderCanonicalNumber(raw) ?? string.Empty, + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + var raw => raw.ToString() ?? string.Empty + }, NumberOf = document => { var raw = getValue(document); @@ -2200,8 +2229,8 @@ namespace GameCult.Caching private readonly List<(CacheBackingStore Store, Type[] Homes)> _stores = new(); private readonly Dictionary _entries = new(StringComparer.Ordinal); private readonly Dictionary> _names = new(); - // Every key holding a value (unordered): the index is truthful about duplicates, so removing one holder never hides another. - private readonly Dictionary<(Type Type, string Alias), Dictionary>> _indexes = new(); + // One key per value: RefuseIndexSharing decides it, Apply only records it. + private readonly Dictionary<(Type Type, string Alias), Dictionary> _indexes = new(); private readonly Dictionary _globals = new(); private readonly CultCodec? _codec; // Keys holding a variant: derived from the entries by Apply, never decided anywhere else. @@ -2671,16 +2700,12 @@ namespace GameCult.Caching .SelectMany(pair => pair.Value.TryGetValue(name, out var key) ? new[] { key } : Array.Empty())); } - // A value held by one record finds it. Records of one type sharing a value (plain duplicates) find the holder with the - // smallest key (ordinal), whatever order they were written or loaded in: the choice is a function of the held set, so - // it is the same live, after a flush and after a reopen, and removing that holder finds the next. + // A [CultIndex] value names one record of its type; blank values are not indexed. public T? GetByIndex(string alias, string value) where T : class { return Single($"index {alias}='{value}'", () => _indexes .Where(pair => pair.Key.Alias == alias && typeof(T).IsAssignableFrom(pair.Key.Type)) - .SelectMany(pair => pair.Value.TryGetValue(value, out var keys) - ? new[] { keys.Aggregate((best, next) => string.CompareOrdinal(next, best) < 0 ? next : best) } - : Array.Empty())); + .SelectMany(pair => pair.Value.TryGetValue(value, out var key) ? new[] { key } : Array.Empty())); } public void Dispose() @@ -3270,12 +3295,10 @@ namespace GameCult.Caching { var plan = ResolveVariants(admitted, evicted, source); Project(plan); - // The variant rules are judged only when a variant is involved. + RefuseIndexSharing(plan, evicted); + // The variant name rule is judged only when a variant is involved. if (_variantKeys.Count != 0 || admitted.Any(stored => stored.Variant != null)) - { RefuseVariantNameSharing(plan); - RefuseVariantIndexSharing(plan, evicted); - } return plan; } @@ -3472,63 +3495,60 @@ namespace GameCult.Caching } } - // The one index rule, judged on the commit's result whatever the write order: a variant may not share an indexed - // value with another record. It reads the truthful held index (every holder of a value), minus the records this - // commit replaces or removes, plus the records it lands. Plain-vs-plain sharing is not this rule's business. - // A unique index maps a value to one key at lookup; a variant sharing it would be found in place of its - // holder or lose the value, and the store would refuse to reopen. - private void RefuseVariantIndexSharing(VariantPlan plan, IReadOnlyList evicted) + // The one index rule, a function of the state the admission would leave and never of staging, file or pull order: + // within one document type and one [CultIndex] alias, no two records hold one non-blank value. The holders of a + // value are the held key (unless this admission replaces or evicts it) plus every landing key projecting it. Over + // every (type, alias, value) held at two or more distinct keys, the refusal names the least by (alias, its least + // key, its second-least key, schema name), each compared by UTF-16 code unit, and never names the value. + private void RefuseIndexSharing(VariantPlan plan, IReadOnlyList evicted) { var landing = plan.Admitted.Concat(plan.Dependents).ToList(); if (landing.All(stored => stored.Descriptor.IndexAccessors.Count == 0)) return; var replaced = new HashSet( plan.Admitted.Concat(evicted).Concat(plan.Dependents).Select(stored => stored.Key.Value), StringComparer.Ordinal); - var landingByKey = landing.ToDictionary(stored => stored.Key.Value, StringComparer.Ordinal); - - bool IsVariant(string key) => - landingByKey.TryGetValue(key, out var landed) ? landed.Variant != null : _variantKeys.Contains(key); - - // One pass over what lands, one lookup per (index, value): the rule is linear in the records landing. - var landedValues = new List<(CultStoredDocument Stored, (Type Type, string Alias) Index, string Value)>(); var landed = new Dictionary<((Type Type, string Alias) Index, string Value), List>(); + var schemaNames = landing.Select(stored => stored.Descriptor).GroupBy(descriptor => descriptor.DocumentType) + .ToDictionary(group => group.Key, group => group.First().SchemaName); foreach (var stored in landing) { foreach (var (alias, value) in plan.Projections[stored.Key.Value].Indexes) { - var index = (stored.Descriptor.DocumentType, alias); - landedValues.Add((stored, index, value)); - if (!landed.TryGetValue((index, value), out var landedKeys)) - landed[(index, value)] = landedKeys = new List(); - landedKeys.Add(stored.Key.Value); + var slot = ((stored.Descriptor.DocumentType, alias), value); + if (!landed.TryGetValue(slot, out var keys)) + landed[slot] = keys = new List(); + keys.Add(stored.Key.Value); } } - var holdersOf = new Dictionary<((Type Type, string Alias) Index, string Value), (string[] All, string[] Variants)>(); - foreach (var (stored, index, value) in landedValues) + (string Alias, string First, string Second, string Schema)? least = null; + foreach (var ((index, value), landedKeys) in landed) { - if (!holdersOf.TryGetValue((index, value), out var known)) - { - var all = (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKeys) - ? heldKeys.Where(heldKey => !replaced.Contains(heldKey)) - : Enumerable.Empty()) - .Concat(landed[(index, value)]) - .Distinct(StringComparer.Ordinal) - .ToArray(); - holdersOf[(index, value)] = known = (all, all.Where(IsVariant).ToArray()); - } - - var self = stored.Key.Value; - var holder = known.All.FirstOrDefault(candidate => candidate != self); - if (stored.Variant != null && holder != null) - throw new InvalidOperationException( - $"Variant {self} and {holder} both hold index {index.Alias} '{value}'; a variant must override its indexed member."); - var variantHolder = known.Variants.FirstOrDefault(candidate => candidate != self); - if (stored.Variant == null && variantHolder != null) - throw new InvalidOperationException( - $"Record {self} and variant {variantHolder} both hold index {index.Alias} '{value}'; " + - "a variant may not share an indexed value with another record."); + var holders = landedKeys.AsEnumerable(); + if (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKey) && !replaced.Contains(heldKey)) + holders = holders.Append(heldKey); + var distinct = holders.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).Take(2).ToArray(); + if (distinct.Length < 2) + continue; + var candidate = (index.Alias, distinct[0], distinct[1], schemaNames[index.Type]); + if (least == null || CompareRefusals(candidate, least.Value) < 0) + least = candidate; } + + if (least is { } pair) + throw new InvalidOperationException( + $"Records {pair.First} and {pair.Second} ({pair.Schema}) " + + $"both hold unique index '{pair.Alias}'; an index value names one record."); + } + + private static int CompareRefusals( + (string Alias, string First, string Second, string Schema) left, + (string Alias, string First, string Second, string Schema) right) + { + var order = StringComparer.Ordinal.Compare(left.Alias, right.Alias); + if (order == 0) order = StringComparer.Ordinal.Compare(left.First, right.First); + if (order == 0) order = StringComparer.Ordinal.Compare(left.Second, right.Second); + return order != 0 ? order : StringComparer.Ordinal.Compare(left.Schema, right.Schema); } private static InvalidOperationException CycleRefusal(IEnumerable keys) => @@ -3633,7 +3653,7 @@ namespace GameCult.Caching if (projection.Name != null) MapOf(_names, type)[projection.Name] = stored.Key.Value; foreach (var (alias, value) in projection.Indexes) - AddHolder(_indexes, (type, alias), value, stored.Key.Value); + MapOf(_indexes, (type, alias))[value] = stored.Key.Value; } // Documents are mutable, so a name or index value may have changed since it was indexed: drop by key. @@ -3645,26 +3665,7 @@ namespace GameCult.Caching if (_names.TryGetValue(type, out var names)) RemoveKey(names, stored.Key.Value); foreach (var pair in _indexes.Where(pair => pair.Key.Type == type)) - RemoveHolder(pair.Value, stored.Key.Value); - } - - private static void AddHolder(Dictionary<(Type Type, string Alias), Dictionary>> indexes, (Type, string) index, string value, string key) - { - if (!indexes.TryGetValue(index, out var map)) - indexes[index] = map = new Dictionary>(StringComparer.Ordinal); - if (!map.TryGetValue(value, out var keys)) - map[value] = keys = new List(); - keys.Remove(key); - keys.Add(key); - } - - private static void RemoveHolder(Dictionary> map, string key) - { - foreach (var pair in map.ToArray()) - { - if (pair.Value.Remove(key) && pair.Value.Count == 0) - map.Remove(pair.Key); - } + RemoveKey(pair.Value, stored.Key.Value); } private static Dictionary MapOf(Dictionary> maps, TKey key) @@ -4648,13 +4649,17 @@ namespace GameCult.Caching /// Whether the record's payload holds element ids, or null when the caller cannot tell. /// The registry the payload was encoded with. Given, the write decides the header for copied records that registry can read. It never decides ownership: the answer is the same lent or not. alone owns the record at the key when it declares the stored id (its own or one it lists compatible), or when it is the type that reads the stored entry and reads it whole, so an older id of its own schema name it reads whole is its own and is restamped. Everything else is refused with the file left as it was: a record of another registered type included, whatever the registry holds, and a record it would read only by dropping persisted slots. Retyping a key is a removal and an add. It serves this call only: the store keeps no registry. /// The call holds the store's gate while it runs. User code it runs must not enter a cache or another store; that throws . - /// The store is attached to a cache; write through the cache. + /// The store is attached to a cache; write through the cache. Also thrown when the record's type declares a , and when the catalog entry declares an index alias and the registry lent for the call does not hold the type (none, or another): the call cannot tell a unique index from a role, so such records are written through a cache. /// The file at the path is not a store this runtime can read. public void WriteRecord(CultPersistedRecord record, CultSchemaCatalogEntry catalogEntry, bool? holdsIds = null, CultDocumentRegistry? registry = null) { if (record == null) throw new ArgumentNullException(nameof(record)); if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry)); - Cacheless(registry, () => WriteOne(record, catalogEntry, holdsIds)); + Cacheless(registry, () => + { + RefuseUniqueIndexedCacheless(AttachedRegistry, catalogEntry); + WriteOne(record, catalogEntry, holdsIds); + }); } /// @@ -4671,7 +4676,7 @@ namespace GameCult.Caching /// The stored-at stamp, or null for now. /// The registry that describes and encodes the document, or null for . The document type's entry is taken from it, and the registry decides the header for copied records it can read. It never decides ownership, so the answer is the same for any registry that holds the type: the document's entry owns a record whose id it declares (its own or one it lists compatible), and a record of an older id of its own schema name that it reads whole; everything else is refused, another registered type's record included, and a record it would read only by dropping persisted slots. Retyping a key is a removal and an add. It serves this call only: the store keeps no registry. /// The call holds the store's gate while it runs. User code it runs (property getters while the document is encoded, constructors while a copied record is read) must not enter a cache or another store; that throws , carried by the encoder's wrapper when the code ran inside it. - /// The store is attached to a cache; write through the cache. + /// The store is attached to a cache; write through the cache. Also thrown when the record's type declares a , and when the catalog entry declares an index alias and the registry lent for the call does not hold the type (none, or another): the call cannot tell a unique index from a role, so such records are written through a cache. /// The file at the path is not a store this runtime can read. /// The document holds an element id or list a cache write refuses. public void WriteDocument(CultRecordKey key, TDocument document, string? storedAt = null, CultDocumentRegistry? registry = null) @@ -4680,6 +4685,7 @@ namespace GameCult.Caching if (document == null) throw new ArgumentNullException(nameof(document)); var resolvedRegistry = registry ?? CultDocumentRegistry.Shared; var catalogEntry = resolvedRegistry.GetRequired().ToCatalogEntry(); + RefuseUniqueIndexedCacheless(resolvedRegistry, catalogEntry); Cacheless(resolvedRegistry, () => { var ids = CultElementIds.Plan(typeof(TDocument), document, key.Value, deterministic: false); @@ -4705,6 +4711,19 @@ namespace GameCult.Caching }); } + // A unique index is judged against every held record, which a cacheless write does not hold. A write is refused when the + // registry lent for the call holds a type with a [CultIndex] under the schema id, and also when the entry declares an index + // alias the call cannot classify (no registry, or one not holding the type): the catalog does not say unique from role. + private static void RefuseUniqueIndexedCacheless(CultDocumentRegistry? registry, CultSchemaCatalogEntry entry) + { + var unique = registry != null && registry.HoldsUniqueIndex(entry.SchemaId); + var unclassified = entry.Members.Any(member => member.IndexAlias != null) && (registry == null || !registry.Declared(entry.SchemaId)); + if (unique || unclassified) + throw new InvalidOperationException( + "A record of a type with a unique index is written through a cache; a cacheless write cannot judge uniqueness" + + (unclassified ? " and the registry lent for the call does not say whether this schema's index is unique." : ".")); + } + private void WriteOne(CultPersistedRecord record, CultSchemaCatalogEntry catalogEntry, bool? holdsIds) { using (AcquireLock(wait: true)) @@ -4727,7 +4746,7 @@ namespace GameCult.Caching RefuseUndeclaredOnDisk(durable, upserts, removals, checks); if (upserts.Count == 0 && removals.Count == 0 && (checks.Any() || StagedChecks.Count > 0)) return Array.Empty<(CultStagedRecord, string)>(); - RefuseWriteIntoMovedVariantStore(durable, upserts, removals); + RefuseWriteIntoMovedStore(durable, upserts, removals); var durableRecords = durable?.Snapshot.Records ?? Array.Empty(); // A record written over one the store holds is stored at a storedAt later than the one it replaces, whoever wrote that. var replaced = durableRecords.ToDictionary(record => record.Key, record => record.StoredAt, StringComparer.Ordinal); @@ -4812,10 +4831,11 @@ namespace GameCult.Caching return false; } - private void RefuseWriteIntoMovedVariantStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) + private void RefuseWriteIntoMovedStore(Durable? durable, IReadOnlyCollection upserts, IReadOnlyCollection removals) { var records = durable?.Snapshot.Records ?? Array.Empty(); - if (records.All(record => record.Variant == null) && upserts.All(staged => staged.Record.Variant == null)) + if (records.All(record => record.Variant == null) && upserts.All(staged => staged.Record.Variant == null) && + !upserts.Any(staged => AttachedRegistry?.HoldsUniqueIndex(staged.Entry.SchemaId) == true)) return; var changed = new HashSet(StringComparer.Ordinal); var versions = records.ToDictionary(record => record.Key, record => durable!.Version(record.Key), StringComparer.Ordinal); @@ -4852,9 +4872,9 @@ namespace GameCult.Caching }, inner => new CultWriteConflictException( Cache == null - ? $"{this} holds a variant and was written by a writer with no CultCache, which cannot resolve variants; nothing was written. " + + ? $"{this} holds a variant or a unique index and was written by a writer with no CultCache, which cannot resolve variants; nothing was written. " + $"Records it would have written or removed: {string.Join(", ", recordKeys.Select(key => $"'{key}'"))}; records that differ from what the writer has read: {string.Join(", ", changedKeys.Select(key => $"'{key}'"))}." - : $"{this} changed since this cache last read it, and it holds a variant, which resolves against other records; nothing was written. " + + : $"{this} changed since this cache last read it, and it holds a variant or a record with a unique index, which resolve against other records; nothing was written. " + $"Records this write would have written or removed: {string.Join(", ", recordKeys.Select(key => $"'{key}'"))}; records that changed: {string.Join(", ", changedKeys.Select(key => $"'{key}'"))}. " + (inner == null ? "The cache reloaded them; retry the write against what the store holds now." diff --git a/src/GameCult.Caching/CultDocumentContracts.cs b/src/GameCult.Caching/CultDocumentContracts.cs index b439ac9f..0f8235ce 100644 --- a/src/GameCult.Caching/CultDocumentContracts.cs +++ b/src/GameCult.Caching/CultDocumentContracts.cs @@ -38,6 +38,7 @@ namespace GameCult.Caching { } + // A unique lookup: within one document type, a non-blank value names one record. Also filterable by CultNet selection. [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)] public sealed class CultIndexAttribute : Attribute { @@ -49,6 +50,19 @@ namespace GameCult.Caching public string? Alias { get; } } + // Names the member's selection and reference role (the alias, or the member name). Implies nothing about uniqueness: + // many records may hold one value, and a role member is never indexed for lookup. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true, AllowMultiple = false)] + public sealed class CultRoleAttribute : Attribute + { + public CultRoleAttribute(string? alias = null) + { + Alias = alias; + } + + public string? Alias { get; } + } + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property, Inherited = true)] public sealed class CultReferenceAttribute : Attribute { diff --git a/src/GameCult.Caching/CultElementIds.cs b/src/GameCult.Caching/CultElementIds.cs index 38dfaaae..97a183e9 100644 --- a/src/GameCult.Caching/CultElementIds.cs +++ b/src/GameCult.Caching/CultElementIds.cs @@ -399,7 +399,7 @@ namespace GameCult.Caching continue; if (shape.Derive == null && !IsRandomId(id!)) throw new CultElementIdException( - $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex; for readable ids, derive them from a member with [CultElementId(nameof(Member))].", + $"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex. To keep an existing readable id, leave that member as it is and add a trailing keyed string member marked [CultElementId(nameof(Member))], where Member is the existing one; the element id is derived from it once and then persisted.", rootPath, path, id); if (!taken.Add(id!)) throw Duplicate(id!, path); diff --git a/src/GameCult.Caching/README.md b/src/GameCult.Caching/README.md index fc5cd308..6defed61 100644 --- a/src/GameCult.Caching/README.md +++ b/src/GameCult.Caching/README.md @@ -22,9 +22,12 @@ public sealed class ItemData [Key(1)] [CultIndex] - public string Category = string.Empty; + public string Sku = string.Empty; [Key(2)] + public string Category = string.Empty; + + [Key(3)] public int Value; } ``` @@ -33,7 +36,8 @@ Available cache attributes: - `CultDocument(schemaName, schemaVersion)` - `CultName` -- `CultIndex(alias?)` +- `CultIndex(alias?)`: a unique lookup. Within one document type, a non-blank value names one record; a commit that would leave two records holding one is refused, and a store holding such a pair refuses to load. Uniqueness is judged by a `CultCache` over the single-file store; a cacheless `WriteDocument`/`WriteRecord` of such a type is refused (also when the catalog declares an index alias the call cannot classify), and concurrent writers on a directory store are not covered (`unique-index-directory-store`). A value is culture-invariant, and a number is its canonical decimal (`1.0m` and `1.00m`, `0.0` and `-0.0` are one value). +- `CultRole(alias?)`: a selection and reference role. Many records may share a value; it is never a lookup. - `CultGlobal` - `CultReference(targetType?, many: false)` @@ -53,6 +57,7 @@ var cache = new CultCache(); var handle = await cache.AddAsync(new ItemData { Name = "Potion", + Sku = "POT-1", Category = "Consumable", Value = 50 }); @@ -66,7 +71,7 @@ Name and index lookups are driven from attributes, not manual registration. ```csharp var byName = cache.GetByName("Potion"); -var byIndex = cache.GetByIndex("Category", "Consumable"); +var byIndex = cache.GetByIndex("Sku", "POT-1"); ``` ### Explicit references @@ -130,6 +135,7 @@ var cache = await CultCacheMessagePack.OpenAsync("Data.msgpack"); var handle = await cache.UpsertAsync(new ItemData { Name = "Potion", + Sku = "POT-1", Category = "Consumable", Value = 50 }); @@ -222,6 +228,60 @@ the text of that member (a string or number on the same type, unique within its list), and a derived id may be any text. A readable name set by hand on a non-derived list id is refused; the same member outside a list is unconstrained. +### A type that already has a readable id + +If the element type already has a readable string id (`Name`, `Slug`, a catalog +key), leave it as it is. Add a trailing keyed string member marked +`[CultElementId(nameof(ExistingId))]`; the element id is derived from the +existing member on load or first write, and then persisted: + +```csharp +[MessagePackObject] +public sealed class Recipe +{ + [Key(0)] public string Name = string.Empty; // existing readable id + [Key(1)] public int Yield; + [Key(2)] [CultElementId(nameof(Name))] public string ElementId = string.Empty; // new, last +} +``` + +Marking `Name` itself `[CultElementId]` is refused (`Name` is not 12 lowercase +hex), and that is deliberate: + +- Minted and hand-written ids share one grammar, 12 lowercase hex, so they can + never collide; a derived id is the one readable form and is checked for + uniqueness within its list. +- An id the consumer can edit is not a stable override address. Variants address + an element by its id, so a field that is renamed in place would silently move + the overrides to a different element, or to none. +- A derived id is persisted once. Renaming `Name` later does not change + `ElementId`, so the element and every override that addresses it stay put. + `ElementId` is the address; `Name` goes back to being ordinary data. + +Adopting the recipe on data that already exists: the existing member becomes an +id source, so it must be non-empty and unique within its list. A record with a +duplicate value and an empty one both refuse to load +(`CultElementIdException`), so clean such data first. + +Take the next free `[Key]` slot, after the highest key on the type (and above +the subclasses' keys on a union base). + +## Store format and rollback + +A store that holds an element id is written as `cultcache.store.v3` (a directory +store as `cultcache.store.v5.directory-content-addressed-pages`, `v4` otherwise). +A single-file store that holds a variant but no id is `v2`, and a store with +neither stays `v1`. The mark follows content: a whole-store write that leaves no +element holding an id drops it. + +Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. + +Rolling a consumer back after it has written a v3 or v5 store is therefore +one-way. The only carry-back is narrow and lossy: a whole-store write that leaves +no id anywhere in the store (every record rewritten without list elements) writes +the store unmarked again, and the ids are gone. Otherwise restore from a copy +taken before upgrading. Upgrade every reader before any writer. + ## Persistence discipline Persistence stays manual by default. Mutations mark the cache and attached diff --git a/src/GameCult.Geometry/CultGeometryDocuments.cs b/src/GameCult.Geometry/CultGeometryDocuments.cs index 169b9ae1..6756d39a 100644 --- a/src/GameCult.Geometry/CultGeometryDocuments.cs +++ b/src/GameCult.Geometry/CultGeometryDocuments.cs @@ -32,7 +32,7 @@ namespace GameCult.Geometry public string DomainId { get; set; } = string.Empty; [Key(1)] - [CultIndex] + [CultRole] public string RootKey { get; set; } = string.Empty; [Key(2)] @@ -164,7 +164,7 @@ namespace GameCult.Geometry public CultRecordRef DomainKey { get; set; } [Key(2)] - [CultIndex] + [CultRole] public string WorkerGroup { get; set; } = string.Empty; [Key(3)] @@ -326,7 +326,7 @@ namespace GameCult.Geometry public CultRecordRef CutKey { get; set; } [Key(2)] - [CultIndex] + [CultRole] public string SelectedCutId { get; set; } = string.Empty; [Key(3)] diff --git a/src/GameCult.Mesh/CultMeshBodyPublications.cs b/src/GameCult.Mesh/CultMeshBodyPublications.cs index 2358c277..86e49365 100644 --- a/src/GameCult.Mesh/CultMeshBodyPublications.cs +++ b/src/GameCult.Mesh/CultMeshBodyPublications.cs @@ -34,7 +34,7 @@ namespace GameCult.Mesh public sealed class CultMeshBodyPublicationDocument { [Key(0), CultName] public string BodyId { get; set; } = string.Empty; - [Key(1), CultIndex] public string ProducerId { get; set; } = string.Empty; + [Key(1), CultRole] public string ProducerId { get; set; } = string.Empty; [Key(2)] public string SchemaId { get; set; } = string.Empty; [Key(3)] public int LayoutVersion { get; set; } [Key(4)] public long ByteSize { get; set; } diff --git a/src/GameCult.Mesh/CultMeshCdn.cs b/src/GameCult.Mesh/CultMeshCdn.cs index 7f47f291..25650f1c 100644 --- a/src/GameCult.Mesh/CultMeshCdn.cs +++ b/src/GameCult.Mesh/CultMeshCdn.cs @@ -142,17 +142,17 @@ namespace GameCult.Mesh /// Artifact kind, such as asset, build, or package. [Key(1)] - [CultIndex] + [CultRole] public string Kind { get; set; } = CultMeshCdnArtifactKinds.Asset; /// Caller-defined artifact version. [Key(2)] - [CultIndex] + [CultRole] public string Version { get; set; } = string.Empty; /// SHA-256 hash of the full materialized artifact. [Key(3)] - [CultIndex] + [CultRole] public string ContentHash { get; set; } = string.Empty; /// Full materialized artifact size. diff --git a/src/GameCult.Mesh/CultMeshEntityPrefabs.cs b/src/GameCult.Mesh/CultMeshEntityPrefabs.cs index afb0b0bf..8d3f26ef 100644 --- a/src/GameCult.Mesh/CultMeshEntityPrefabs.cs +++ b/src/GameCult.Mesh/CultMeshEntityPrefabs.cs @@ -193,12 +193,12 @@ namespace GameCult.Mesh /// Caller-defined package version. [Key(1)] - [CultIndex] + [CultRole] public string Version { get; set; } = string.Empty; /// Tool that authored the package, such as Brokkr Blender. [Key(2)] - [CultIndex] + [CultRole] public string SourceTool { get; set; } = string.Empty; /// Source scene name or path. @@ -207,12 +207,12 @@ namespace GameCult.Mesh /// Source Blender collection name. [Key(4)] - [CultIndex] + [CultRole] public string SourceCollection { get; set; } = string.Empty; /// Stable hash of graph structure and referenced assets. [Key(5)] - [CultIndex] + [CultRole] public string ContentHash { get; set; } = string.Empty; /// Package creation timestamp in round-trip UTC format. diff --git a/src/GameCult.Mesh/CultMeshSimulationFacts.cs b/src/GameCult.Mesh/CultMeshSimulationFacts.cs index 11c7d296..562be44a 100644 --- a/src/GameCult.Mesh/CultMeshSimulationFacts.cs +++ b/src/GameCult.Mesh/CultMeshSimulationFacts.cs @@ -20,14 +20,14 @@ namespace GameCult.Mesh /// Stable fact identifier. /// [Key(0)] - [CultIndex] + [CultRole] public string FactId { get; set; } = string.Empty; /// /// Shard containing the simulation. /// [Key(1)] - [CultIndex] + [CultRole] public string ShardId { get; set; } = string.Empty; /// @@ -40,21 +40,21 @@ namespace GameCult.Mesh /// Simulation frame or tick. /// [Key(3)] - [CultIndex] + [CultRole] public long Frame { get; set; } /// /// Subject the fact is about. /// [Key(4)] - [CultIndex] + [CultRole] public string SubjectId { get; set; } = string.Empty; /// /// Fact kind, such as hit, collision, or ownership. /// [Key(5)] - [CultIndex] + [CultRole] public string ClaimKind { get; set; } = string.Empty; /// diff --git a/src/GameCult.Mesh/docs/public-api.md b/src/GameCult.Mesh/docs/public-api.md index 072e6c73..31737e9c 100644 --- a/src/GameCult.Mesh/docs/public-api.md +++ b/src/GameCult.Mesh/docs/public-api.md @@ -321,15 +321,16 @@ var bySchema = documents.DocumentBySchema("gamecult.aetheria.station_refit.v1"); Typed collections use the same one-call style: ```csharp -var allies = CultMesh.CollectionByIndex( +// A [CultIndex] value is unique within its type, so the handle holds at most one record. +var flagship = CultMesh.CollectionByIndex( node, - "Faction", - "au"); + "CallSign", + "au-1"); -var currentAllies = await allies.LatestAsync(); -using var changes = allies.WatchChanges(change => UpdateRoster(change)); +var currentFlagship = await flagship.LatestAsync(); +using var changes = flagship.WatchChanges(change => UpdateRoster(change)); -var collections = CultMesh.Collections(allies); +var collections = CultMesh.Collections(flagship); var roster = await collections.LatestAsync(); using var rosterChanges = collections.WatchChanges(RenderRoster); ``` diff --git a/src/GameCult.Networking/CultNetDatabase.cs b/src/GameCult.Networking/CultNetDatabase.cs index af561ea5..9db3cd1b 100644 --- a/src/GameCult.Networking/CultNetDatabase.cs +++ b/src/GameCult.Networking/CultNetDatabase.cs @@ -315,6 +315,22 @@ namespace GameCult.Networking public CultNetSelectionCursorKey? CursorKey { get; set; } } + /// + /// The typed changes ( boxed) one cache commit made, in the order the cache admitted them. + /// is opaque and unique among the commits of one database instance. + /// + public sealed class CultNetDatabaseCommit + { + public CultNetDatabaseCommit(string id, IReadOnlyList changes) + { + Id = id; + Changes = changes; + } + + public string Id { get; } + public IReadOnlyList Changes { get; } + } + /// /// A typed domain change emitted by a CultNet database surface. /// @@ -587,6 +603,7 @@ namespace GameCult.Networking private readonly Dictionary<(string SchemaId, string RecordKey), long> _lastWriteSequence = new(); private readonly Subject _changes = new(); + private readonly Subject _commits = new(); private readonly IDisposable _cacheChanges; private readonly IDisposable _cacheJournal; // What a write door of this database knows about the change it is admitting and the cache does not: the wire @@ -1148,6 +1165,17 @@ namespace GameCult.Networking return _changes; } + /// + /// Watches changes as the cache commits that made them: one event per commit, after every change of it was published to + /// . A consumer that forwards changes to a replica forwards a commit as a unit, so the replica + /// can judge the result of the commit and not each record's intermediate state. + /// + public Observable WatchCommits() + { + ThrowIfDisposed(); + return _commits; + } + /// /// Watches one record. /// @@ -1215,6 +1243,7 @@ namespace GameCult.Networking } _changes.Dispose(); + _commits.Dispose(); } // The cache is the one owner of "a change was committed"; the journal is this database's one decision point and its @@ -1231,6 +1260,8 @@ namespace GameCult.Networking } List? failures = null; + // One journal call is one admission, so one commit; its first Sequence names it. + var commit = new CommitGroup(changes.Count == 0 ? "0" : changes[0].Sequence.ToString(System.Globalization.CultureInfo.InvariantCulture)); List? burned = null; List? burnCauses = null; foreach (var change in changes) @@ -1288,7 +1319,10 @@ namespace GameCult.Networking } lock (_logGate) - _stash[change.Sequence] = new Publication(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument); + { + _stash[change.Sequence] = new Publication(documentType, kind, change.Key, descriptor.SchemaId, shard, change.Document, change.PreviousDocument, commit); + commit.Unpublished++; + } } // Every change is stashed and will be published; the writer hears of a failed log afterwards, in one exception. @@ -1321,8 +1355,9 @@ namespace GameCult.Networking Publish(publication); } - private void Publish(Publication publication) => - PublishUntyped( + private void Publish(Publication publication) + { + var change = PublishUntyped( publication.DocumentType, publication.Kind, publication.Key, @@ -1330,6 +1365,19 @@ namespace GameCult.Networking publication.Shard, publication.Document, publication.PreviousDocument); + CultNetDatabaseCommit? finished = null; + lock (_logGate) + { + var commit = publication.Commit; + if (change != null) + commit.Published.Add(change); + if (--commit.Unpublished == 0 && commit.Published.Count > 0) + finished = new CultNetDatabaseCommit(commit.Id, commit.Published.ToArray()); + } + + if (finished != null) + _commits.OnNext(finished); + } private void RecordReplicatedEntry( CultNetShardDescriptor shard, @@ -1467,8 +1515,10 @@ namespace GameCult.Networking string schemaId, CultNetShardDescriptor shard, object? document, - object? previousDocument) + object? previousDocument, + CommitGroup commit) { + Commit = commit; DocumentType = documentType; Kind = kind; Key = key; @@ -1485,6 +1535,16 @@ namespace GameCult.Networking public CultNetShardDescriptor Shard { get; } public object? Document { get; } public object? PreviousDocument { get; } + public CommitGroup Commit { get; } + } + + // The changes of one cache admission, counted down as they are published. Guarded by _logGate. + private sealed class CommitGroup + { + public CommitGroup(string id) => Id = id; + public string Id { get; } + public int Unpublished { get; set; } + public List Published { get; } = new(); } private sealed class DoorContext @@ -1964,7 +2024,7 @@ namespace GameCult.Networking return $"{schemaId}:{key.Value}"; } - private void PublishUntyped( + private object? PublishUntyped( Type documentType, CultNetDatabaseChangeKind kind, CultRecordKey key, @@ -1987,6 +2047,8 @@ namespace GameCult.Networking { _changes.OnNext(change); } + + return change; } // No shard owns a schema and key that no descriptor matches, and nothing is written to one: every write path refuses. diff --git a/src/GameCult.Networking/CultNetDatabaseServer.cs b/src/GameCult.Networking/CultNetDatabaseServer.cs index e1694131..32f302fc 100644 --- a/src/GameCult.Networking/CultNetDatabaseServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseServer.cs @@ -765,14 +765,41 @@ namespace GameCult.Networking { // Keys this subscriber only ever received as a refusal: it was never given them as records. var refused = new HashSet(StringComparer.Ordinal); - return _database.WatchAllChanges().Subscribe(change => + return _database.WatchCommits().Subscribe(commit => { - var outbound = DeliverChange(change, subscriptionId, selection, refused, error => peer.SendCultNet(error)); - if (outbound != null) - peer.SendCultNet(outbound); + foreach (var message in DeliverCommit(commit, subscriptionId, selection, refused, error => peer.SendCultNet(error))) + peer.SendCultNet(message); }); } + // One subscriber's view of one commit. Its changes go out together, tagged with the commit, so a replica applies them as one. + internal IReadOnlyList DeliverCommit( + CultNetDatabaseCommit commit, + string subscriptionId, + CultNetSelection selection, + HashSet refused, + Action sendError) + { + var outbound = new List(); + foreach (var change in commit.Changes) + { + var message = DeliverChange(change, subscriptionId, selection, refused, sendError); + if (message != null) + outbound.Add(message); + } + + if (outbound.Count > 1) + { + foreach (var message in outbound) + { + message.CommitId = commit.Id; + message.CommitCount = outbound.Count; + } + } + + return outbound; + } + // One subscriber's view of one change. A variant is not a record CultNet can carry (Q6): the subscriber is told, by key, and // is not later told the variant was removed, because it never held it. internal CultNetDatabaseChangeRawMessage? DeliverChange( diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs index db1e2907..bf3772e6 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionClient.cs @@ -29,6 +29,14 @@ namespace GameCult.Networking private readonly ConcurrentDictionary _initialSnapshots = new(); private readonly ConcurrentDictionary _deliveryModes = new(StringComparer.Ordinal); + // Changes of a commit that has not fully arrived, by subscription and commit id. + // Guarded by its own lock: the queue fills it, Unsubscribe and Dispose evict. A commit that does not complete is bounded + // by MaxPendingCommits and by MaxBufferedChanges, and a violation faults the client. + private const int MaxPendingCommits = 64; + // Policy: the most changes the incomplete commits may announce between them, the bound on what the buffer can hold. It + // sits far above any commit a source makes; a commit that announces more than this could not be held at all. + private const int MaxBufferedChanges = 1_000_000; + private readonly Dictionary<(string SubscriptionId, string CommitId), PendingCommit> _commits = new(); private readonly object _queueLock = new(); private Task _queue = Task.CompletedTask; private bool _disposed; @@ -48,6 +56,15 @@ namespace GameCult.Networking /// Raised after a live change has been applied to the local typed cache. public event Action? Changed; + /// + /// Raised when the local cache refuses a replicated commit. The replica then differs from its source; the client applies + /// no further changes and reports until it is replaced by a new subscription. + /// + public event Action? Faulted; + + /// The refusal that stopped replication, or null while the replica is intact. + public Exception? Fault { get; private set; } + /// Gets the local reactive cache receiving replicated documents. public CultCache Cache => _cache; @@ -109,6 +126,8 @@ namespace GameCult.Networking CultNetDatabaseSubscriptionDeliveryMode deliveryMode = CultNetDatabaseSubscriptionDeliveryMode.ReplicateToCache) { if (_disposed) throw new ObjectDisposedException(nameof(CultNetDatabaseSubscriptionClient)); + if (Fault != null) + throw new InvalidOperationException("The replica refused a commit and differs from its source; subscribe with a new client.", Fault); if (!_client.Connected) throw new InvalidOperationException("CultNet schema client must be connected before subscribing."); if (string.IsNullOrWhiteSpace(subscriptionId)) throw new ArgumentException("Subscription id is required.", nameof(subscriptionId)); @@ -166,6 +185,9 @@ namespace GameCult.Networking if (_disposed) return; if (string.IsNullOrWhiteSpace(subscriptionId)) throw new ArgumentException("Subscription id is required.", nameof(subscriptionId)); _deliveryModes.TryRemove(subscriptionId, out _); + lock (_commits) + foreach (var held in _commits.Keys.Where(key => key.SubscriptionId == subscriptionId).ToArray()) + _commits.Remove(held); _client.SendCultNet(new CultNetDatabaseUnsubscribeMessage { MessageId = Guid.NewGuid().ToString("N"), @@ -181,6 +203,8 @@ namespace GameCult.Networking pending.Completion.TrySetException(new ObjectDisposedException(nameof(CultNetDatabaseSubscriptionClient))); _initialSnapshots.Clear(); _deliveryModes.Clear(); + lock (_commits) + _commits.Clear(); _client.Dispose(); } @@ -211,48 +235,110 @@ namespace GameCult.Networking private void HandleChange(CultNetDatabaseChangeRawMessage message) { - Enqueue(async () => + Enqueue(() => { var deliveryMode = _deliveryModes.TryGetValue(message.SubscriptionId, out var configured) ? configured : CultNetDatabaseSubscriptionDeliveryMode.ReplicateToCache; - if (string.Equals(message.ChangeKind, "removed", StringComparison.Ordinal)) + // Live delivery applies nothing, so there is no commit to judge; each change is reported as it arrives. + if (deliveryMode != CultNetDatabaseSubscriptionDeliveryMode.ReplicateToCache || message.CommitId == null || !(message.CommitCount > 1)) + return Apply(deliveryMode, new[] { message }); + + // The commit's other changes follow on this connection; the replica applies them as one commit. + var held = (message.SubscriptionId, message.CommitId); + PendingCommit? pending; + lock (_commits) { - if (deliveryMode == CultNetDatabaseSubscriptionDeliveryMode.ReplicateToCache) - await DeleteAsync(message).ConfigureAwait(false); - Changed?.Invoke(new CultNetReplicatedDocumentChange( - message.SubscriptionId, - "removed", - message.RecordKey ?? string.Empty, - message.SchemaId ?? string.Empty, - null)); - return; + // A straggler of a subscription that was ended, or never made, is not buffered. Judged under the lock that + // Unsubscribe evicts under: a check before it could pass, lose the race to the eviction, and then buffer. + if (Fault != null || !_deliveryModes.ContainsKey(message.SubscriptionId)) + return Task.CompletedTask; + var count = message.CommitCount!.Value; + var refusal = !_commits.TryGetValue(held, out pending) + ? (_commits.Count >= MaxPendingCommits + ? $"more than {MaxPendingCommits} commits are incomplete at once" + : (long)count + _commits.Sum(entry => (long)entry.Value.Announced) > MaxBufferedChanges + ? $"commit '{message.CommitId}' announces {count} changes; the incomplete commits may announce {MaxBufferedChanges} between them" + : null) + : pending.Announced != count + ? $"commit '{message.CommitId}' announces {pending.Announced} and {count} changes" + : pending.Ids.Contains(message.MessageId) + ? $"commit '{message.CommitId}' delivered message '{message.MessageId}' twice" + : null; + if (refusal != null) + { + Break(new InvalidOperationException($"CultNet subscription '{message.SubscriptionId}': {refusal}.")); + return Task.CompletedTask; + } + + if (!_commits.TryGetValue(held, out pending)) + _commits[held] = pending = new PendingCommit(count); + pending.Ids.Add(message.MessageId); + pending.Messages.Add(message); + if (pending.Messages.Count < count) + return Task.CompletedTask; + _commits.Remove(held); } - if (message.Document == null) return; - var document = deliveryMode == CultNetDatabaseSubscriptionDeliveryMode.Live - ? _documents.DeserializeRawDocument(message.Document) - : await _documents.ApplyRawDocumentPutMessageAsync( - _cache, - new CultNetDocumentPutRawMessage - { - MessageId = message.MessageId, - Document = message.Document - }).ConfigureAwait(false); - Changed?.Invoke(new CultNetReplicatedDocumentChange( - message.SubscriptionId, - message.ChangeKind, - message.Document.RecordKey, - message.Document.SchemaId, - document)); + return Apply(deliveryMode, pending.Messages); }); } - private Task DeleteAsync(CultNetDatabaseChangeRawMessage message) + private sealed class PendingCommit { - if (string.IsNullOrWhiteSpace(message.RecordKey) || string.IsNullOrWhiteSpace(message.SchemaId)) return Task.CompletedTask; - _documents.ResolveDescriptorForSchemaId(message.SchemaId!); - _cache.Remove(new CultRecordKey(message.RecordKey!)); + public PendingCommit(int announced) => Announced = announced; + public int Announced { get; } + public List Messages { get; } = new(); + public HashSet Ids { get; } = new(StringComparer.Ordinal); + } + + // The replica no longer matches its source: say so once, drop what was waiting, and apply nothing more. + private void Break(Exception error) + { + lock (_commits) + { + if (Fault != null) + return; + Fault = error; + _commits.Clear(); + } + + Faulted?.Invoke(error); + } + + private Task Apply(CultNetDatabaseSubscriptionDeliveryMode deliveryMode, IReadOnlyList changes) + { + // A replica that refused a commit no longer matches its source. Applying later commits on top would hide that, so + // nothing more is applied: the fault is surfaced and the replica is rebuilt by subscribing again. + if (Fault != null) + return Task.CompletedTask; + IReadOnlyList applied; + try + { + applied = deliveryMode == CultNetDatabaseSubscriptionDeliveryMode.Live + ? changes.Select(change => change.Document == null ? null : _documents.DeserializeRawDocument(change.Document)).ToArray() + : _documents.ApplyRawChangesAsOneCommit(_cache, changes); + } + catch (Exception error) + { + Break(error); + return Task.CompletedTask; + } + + for (var index = 0; index < changes.Count; index++) + { + var change = changes[index]; + var removed = string.Equals(change.ChangeKind, "removed", StringComparison.Ordinal); + if (!removed && change.Document == null) + continue; + Changed?.Invoke(new CultNetReplicatedDocumentChange( + change.SubscriptionId, + change.ChangeKind, + removed ? change.RecordKey ?? string.Empty : change.Document!.RecordKey, + removed ? change.SchemaId ?? string.Empty : change.Document!.SchemaId, + removed ? null : applied[index])); + } + return Task.CompletedTask; } diff --git a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs index 8398feba..3e23cf10 100644 --- a/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs +++ b/src/GameCult.Networking/CultNetDatabaseSubscriptionServer.cs @@ -77,7 +77,20 @@ namespace GameCult.Networking { if (_disposed) return; foreach (var entry in _requests.ToArray()) - Reconcile(entry.Key, entry.Value); + { + // Reconcile can run after the cache has admitted a commit and before Watch has seen it, so its diff can carry + // changed values; it is held and sent as one commit of its own, as Watch's is, for the replica to judge whole. + var outbox = _outbox = new CommitOutbox(Guid.NewGuid().ToString("N")); + try + { + Reconcile(entry.Key, entry.Value); + } + finally + { + _outbox = null; + outbox.Send(entry.Key.Peer); + } + } } } @@ -300,50 +313,76 @@ namespace GameCult.Networking SubscriptionKey key) { var legacyRequest = request.ToLegacyShape(subscriptionId); - return _database.WatchAllChanges().Subscribe(change => + return _database.WatchCommits().Subscribe(commit => { lock (_lifecycleGate) { if (_disposed || !_requests.ContainsKey(key) || !_projections.TryGetValue(key, out var projection)) return; - if (_authorizeRequest?.Invoke(legacyRequest, peer) == false) - { - Reconcile(key, request); - return; - } - - // D6: cites/cited are set-dependent - a change to row B can change whether row A - // matches - so a hop-bearing selection runs the full diff-against-delivered - // reconcile on every change instead of the single-row fast path. - if (request.Selection.HasHop) - { - Reconcile(key, request); - return; - } - - var sourceRecordKey = ResolveChangeRecordKey(change); - CultNetRawDocumentRecord? matched; + // What this subscription sends for the commit is held and sent together, tagged with the commit, so the replica + // applies it as one commit and judges the result. + var outbox = _outbox = new CommitOutbox(commit.Id); try { - matched = CreateMatchedRecord(change, request, peer); + ForwardCommit(commit, request, key, projection, legacyRequest, peer, subscriptionId); } - catch (NotSupportedException refusal) + finally { - // Q6: a variant is not a record CultNet can carry; the subscriber that would have received it is told, by key. - peer.SendCultNet(CultNetErrorMessage.ForVariantUnsupported(refusal.Message)); - return; + _outbox = null; + outbox.Send(peer); } - - ApplyProjectedChange( - projection, - sourceRecordKey, - matched, - peer, - subscriptionId); } }); } + private void ForwardCommit( + CultNetDatabaseCommit commit, + SubscriptionRequest request, + SubscriptionKey key, + SubscriptionProjectionState projection, + CultNetDatabaseSubscribeMessage legacyRequest, + ICultNetSchemaServerPeer peer, + string subscriptionId) + { + if (_authorizeRequest?.Invoke(legacyRequest, peer) == false) + { + Reconcile(key, request); + return; + } + + // D6: cites/cited are set-dependent - a change to row B can change whether row A + // matches - so a hop-bearing selection runs the full diff-against-delivered + // reconcile once per commit instead of the single-row fast path. + if (request.Selection.HasHop) + { + Reconcile(key, request); + return; + } + + foreach (var change in commit.Changes) + { + var sourceRecordKey = ResolveChangeRecordKey(change); + CultNetRawDocumentRecord? matched; + try + { + matched = CreateMatchedRecord(change, request, peer); + } + catch (NotSupportedException refusal) + { + // Q6: a variant is not a record CultNet can carry; the subscriber that would have received it is told, by key. + peer.SendCultNet(CultNetErrorMessage.ForVariantUnsupported(refusal.Message)); + continue; + } + + ApplyProjectedChange( + projection, + sourceRecordKey, + matched, + peer, + subscriptionId); + } + } + private void Reconcile(SubscriptionKey key, SubscriptionRequest request) { if (!_projections.TryGetValue(key, out var projection)) return; @@ -487,12 +526,15 @@ namespace GameCult.Networking string.Equals(left.SourceAgentId, right.SourceAgentId, StringComparison.Ordinal) && string.Equals(left.SourceRole, right.SourceRole, StringComparison.Ordinal); - private static void SendUpsert( + // The commit being forwarded, under _lifecycleGate: the change messages it produces are held here, not sent one by one. + private CommitOutbox? _outbox; + + private void SendUpsert( ICultNetSchemaServerPeer peer, string subscriptionId, CultNetRawDocumentRecord document, bool added) => - peer.SendCultNet(new CultNetDatabaseChangeRawMessage + Deliver(peer, new CultNetDatabaseChangeRawMessage { MessageId = Guid.NewGuid().ToString("N"), SubscriptionId = subscriptionId, @@ -500,11 +542,11 @@ namespace GameCult.Networking Document = document }); - private static void SendRemoval( + private void SendRemoval( ICultNetSchemaServerPeer peer, string subscriptionId, CultNetRawDocumentRecord document) => - peer.SendCultNet(new CultNetDatabaseChangeRawMessage + Deliver(peer, new CultNetDatabaseChangeRawMessage { MessageId = Guid.NewGuid().ToString("N"), SubscriptionId = subscriptionId, @@ -513,6 +555,38 @@ namespace GameCult.Networking SchemaId = document.SchemaId }); + private void Deliver(ICultNetSchemaServerPeer peer, CultNetDatabaseChangeRawMessage message) + { + if (_outbox is { } outbox) + outbox.Messages.Add(message); + else + peer.SendCultNet(message); + } + + private sealed class CommitOutbox + { + private readonly string _commitId; + + public CommitOutbox(string commitId) => _commitId = commitId; + + public List Messages { get; } = new(); + + public void Send(ICultNetSchemaServerPeer peer) + { + if (Messages.Count > 1) + { + foreach (var message in Messages) + { + message.CommitId = _commitId; + message.CommitCount = Messages.Count; + } + } + + foreach (var message in Messages) + peer.SendCultNet(message); + } + } + private void Withdraw(SubscriptionKey key, bool sendRemovals, bool forgetRequest) { if (_subscriptions.TryRemove(key, out var subscription)) diff --git a/src/GameCult.Networking/CultNetDocumentRegistry.cs b/src/GameCult.Networking/CultNetDocumentRegistry.cs index 58b7ca36..19f17c8c 100644 --- a/src/GameCult.Networking/CultNetDocumentRegistry.cs +++ b/src/GameCult.Networking/CultNetDocumentRegistry.cs @@ -722,6 +722,49 @@ namespace GameCult.Networking return document; } + /// + /// Applies change messages to a cache as one commit, so the cache judges the result of the changes together and not the + /// state between them. Returns the applied document of each message, null for a removal. A refusal by the cache throws + /// and nothing is applied. + /// + internal IReadOnlyList ApplyRawChangesAsOneCommit( + CultCache cache, + IReadOnlyList changes) + { + var applied = new object?[changes.Count]; + var staged = new List<(Type? Type, object? Document, CultRecordKey Key)>(changes.Count); + for (var index = 0; index < changes.Count; index++) + { + var change = changes[index]; + if (string.Equals(change.ChangeKind, "removed", StringComparison.Ordinal)) + { + if (string.IsNullOrWhiteSpace(change.RecordKey) || string.IsNullOrWhiteSpace(change.SchemaId)) + continue; + ResolveDescriptorForSchemaId(change.SchemaId!); + staged.Add((null, null, new CultRecordKey(change.RecordKey!))); + continue; + } + + if (change.Document == null) + continue; + var document = DeserializeRawDocument(change.Document); + applied[index] = document; + staged.Add((ResolveDescriptorForRawDocument(change.Document).DocumentType, document, new CultRecordKey(change.Document.RecordKey))); + } + + cache.Commit(batch => + { + foreach (var (type, document, key) in staged) + { + if (type == null) + batch.Remove(key); + else + batch.Upsert(type, document!, key); + } + }); + return applied; + } + /// /// Decodes one validated raw document without materializing it in a CultCache replica. /// Use this for ephemeral reactive subscriptions whose callback is the consumer. @@ -754,25 +797,22 @@ namespace GameCult.Networking } /// - /// Applies all documents from a raw snapshot response. + /// Applies all documents from a raw snapshot response as one commit, so a snapshot that moves a unique-index value + /// between records is judged on its result. /// - public async Task> ApplyRawSnapshotResponseAsync( + public Task> ApplyRawSnapshotResponseAsync( CultCache cache, CultNetSnapshotResponseRawMessage response) { - var applied = new List(response.Documents.Length); - foreach (var document in response.Documents) - { - applied.Add(await ApplyRawDocumentPutMessageAsync( - cache, - new CultNetDocumentPutRawMessage - { - MessageId = response.MessageId, - Document = document - }).ConfigureAwait(false)); - } - - return applied; + var applied = ApplyRawChangesAsOneCommit( + cache, + response.Documents.Select(document => new CultNetDatabaseChangeRawMessage + { + SubscriptionId = response.MessageId, + ChangeKind = "added", + Document = document + }).ToArray()); + return Task.FromResult>(applied.Select(document => document!).ToArray()); } internal CultDocumentDescriptor ResolveDescriptorForRawDocument(CultNetRawDocumentRecord document) diff --git a/src/GameCult.Networking/CultNetSchemaMessages.cs b/src/GameCult.Networking/CultNetSchemaMessages.cs index a2211672..bc59b007 100644 --- a/src/GameCult.Networking/CultNetSchemaMessages.cs +++ b/src/GameCult.Networking/CultNetSchemaMessages.cs @@ -1179,6 +1179,17 @@ namespace GameCult.Networking /// Gets or sets the deleted schema id for removed changes. /// [Key("schemaId")] public string? SchemaId { get; set; } + /// + /// Gets or sets the identity of the cache commit this change belongs to. Optional and additive: a peer that does not + /// know the field ignores it and applies the change alone. Opaque to the receiver; equal ids on one subscription + /// name changes of one commit. + /// + [Key("commitId")] public string? CommitId { get; set; } + /// + /// Gets or sets how many change messages of this commit this subscription carries. Set with + /// when that is more than one; the receiver applies the commit once all have arrived. + /// + [Key("commitCount")] public int? CommitCount { get; set; } } /// diff --git a/tests/GameCult.Caching.Tests/CultDocumentSelectionSurfaceTests.cs b/tests/GameCult.Caching.Tests/CultDocumentSelectionSurfaceTests.cs index daa71c81..ed77bb90 100644 --- a/tests/GameCult.Caching.Tests/CultDocumentSelectionSurfaceTests.cs +++ b/tests/GameCult.Caching.Tests/CultDocumentSelectionSurfaceTests.cs @@ -226,8 +226,8 @@ namespace GameCult.Caching.Tests public void DuplicateIndexAliasOnTwoDifferentlyNamedMembersIsRefusedByName() { var type = Document("DupAlias"); - Field(type, "First", typeof(string), Key(0), IndexAttr("shared")); - Field(type, "Second", typeof(string), Key(1), IndexAttr("shared")); + Field(type, "First", typeof(string), Key(0), RoleAttr("shared")); + Field(type, "Second", typeof(string), Key(1), RoleAttr("shared")); var message = Assert.Throws(() => CultDocumentRegistry.ForTypes(new[] { type.CreateType()! }))!.Message; Assert.That(message, Does.Contain("DupAlias")); @@ -236,7 +236,7 @@ namespace GameCult.Caching.Tests Assert.That(message, Does.Contain("shared")); } - // R-AL/SM-6: a plain data member's bare name (it declares no [CultIndex], so its role in + // R-AL/SM-6: a plain data member's bare name (it declares no [CultRole], so its role in // ReferencesOf/ReferenceMembers falls back to its own MemberName) colliding with a reference // member's explicit alias. D10 above only groups members that declare an alias and misses // this; before the fix this registered without error and then every hop through the @@ -246,7 +246,7 @@ namespace GameCult.Caching.Tests { var type = Document("OwnerCollision"); Field(type, "Owner", typeof(string), Key(0)); - Field(type, "Ref", typeof(CultRecordRef), Key(1), IndexAttr("Owner")); + Field(type, "Ref", typeof(CultRecordRef), Key(1), RoleAttr("Owner")); var message = Assert.Throws(() => CultDocumentRegistry.ForTypes(new[] { type.CreateType()! }))!.Message; Assert.That(message, Does.Contain("OwnerCollision")); @@ -295,8 +295,8 @@ namespace GameCult.Caching.Tests private static CustomAttributeBuilder Key(int slot) => new(typeof(KeyAttribute).GetConstructor(new[] { typeof(int) })!, new object[] { slot }); - private static CustomAttributeBuilder IndexAttr(string alias) => - new(typeof(CultIndexAttribute).GetConstructor(new[] { typeof(string) })!, new object?[] { alias }); + private static CustomAttributeBuilder RoleAttr(string alias) => + new(typeof(CultRoleAttribute).GetConstructor(new[] { typeof(string) })!, new object?[] { alias }); private static CustomAttributeBuilder ReferenceAttr(Type? targetType, bool many) => new(typeof(CultReferenceAttribute).GetConstructor(new[] { typeof(Type), typeof(bool) })!, new object?[] { targetType, many }); @@ -308,7 +308,7 @@ namespace GameCult.Caching.Tests public string Name = string.Empty; [Key(1)] - [CultIndex("mass")] + [CultRole("mass")] public float Mass; } @@ -325,7 +325,7 @@ namespace GameCult.Caching.Tests public sealed class FixtureLeafB : FixtureMiddle { [Key(2)] - [CultIndex("weight")] + [CultRole("weight")] public decimal? Weight; } @@ -341,19 +341,19 @@ namespace GameCult.Caching.Tests public string Name = string.Empty; [Key(1)] - [CultIndex("big_integer")] + [CultRole("big_integer")] public long BigInteger; [Key(2)] - [CultIndex("trimmed_decimal")] + [CultRole("trimmed_decimal")] public decimal TrimmedDecimal; [Key(3)] - [CultIndex("wide_double")] + [CultRole("wide_double")] public double WideDouble; [Key(4)] - [CultIndex("tiny_double")] + [CultRole("tiny_double")] public double TinyDouble; } diff --git a/tests/GameCult.Caching.Tests/DocumentShapeTests.cs b/tests/GameCult.Caching.Tests/DocumentShapeTests.cs index c34834ca..0d473a5a 100644 --- a/tests/GameCult.Caching.Tests/DocumentShapeTests.cs +++ b/tests/GameCult.Caching.Tests/DocumentShapeTests.cs @@ -394,7 +394,7 @@ namespace GameCult.Caching.Tests public class Gear { [Key(0)] [CultName] public string Name { get; set; } = ""; - [Key(1)] [CultIndex("slot")] public string Slot { get; set; } = ""; + [Key(1)] [CultRole("slot")] public string Slot { get; set; } = ""; } [CultDocument("shape.weapon", "shape.weapon.v1")] diff --git a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs index 2bc1d9df..a54251ed 100644 --- a/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdEngineTests.cs @@ -142,7 +142,7 @@ namespace GameCult.Caching.Tests public void TheHexRefusalNamesTheDerivedFormForReadableIds() { var refusal = Assert.Throws(() => Assign(new HHasIds { Items = { new EIded { Id = "root" } } }, "r", deterministic: false))!; - Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]")); + Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]").And.Contain("trailing keyed string member").And.Contain("existing readable id")); } [Test] diff --git a/tests/GameCult.Caching.Tests/ElementIdTests.cs b/tests/GameCult.Caching.Tests/ElementIdTests.cs index 1b6b3fe2..a6c68c94 100644 --- a/tests/GameCult.Caching.Tests/ElementIdTests.cs +++ b/tests/GameCult.Caching.Tests/ElementIdTests.cs @@ -389,6 +389,44 @@ namespace GameCult.Caching.Tests Assert.That(refusal.Message, Does.Contain("'7'")); } + // The documented recipe for a type that already has a readable id (src/GameCult.Caching/README.md, Element ids). + [Test] + public void ATrailingMemberDerivedFromAnExistingReadableIdIsPersistedOnceAndTheStoreIsMarkedV3() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RecipeBook) }); + var path = PathOf("recipes.cc"); + var key = new CultRecordKey("book"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook { Recipes = { new IdRecipe { Name = "bread" }, new IdRecipe { Name = "soup" } } }, key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry })) + { + var book = cache.Get(key)!; + Assert.That(book.Recipes.Select(recipe => recipe.ElementId), Is.EqualTo(new[] { "bread", "soup" })); + book.Recipes[0].Name = "loaf"; + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), book, key)); + } + + using var again = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + var reread = again.Get(key)!; + Assert.That(reread.Recipes.Select(recipe => recipe.Name), Is.EqualTo(new[] { "loaf", "soup" })); + Assert.That(reread.Recipes.Select(recipe => recipe.ElementId), Is.EqualTo(new[] { "bread", "soup" }), "renaming the source does not move the element"); + } + + [Test] + public void AWholeStoreWriteThatLeavesNoIdInTheStoreDropsTheMarkBackToV1() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RecipeBook) }); + var path = PathOf("drop.cc"); + var key = new CultRecordKey("book"); + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook { Recipes = { new IdRecipe { Name = "bread" } } }, key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3")); + cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook(), key)); + Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the mark follows content: no element holds an id, so the store is unmarked"); + } + // ---- C2a fix batch: the marker, refusals at load, override values, atomic minting ---- private const string HexA = "aaaaaaaaaaaa"; @@ -1359,6 +1397,20 @@ namespace GameCult.Caching.Tests [Key(1)] public List Labels { get; set; } = new(); } + [CultDocument("tests.element_id_recipe_book", "tests.element_id_recipe_book.v1")] + [MessagePackObject] + public sealed class RecipeBook + { + [Key(0)] public List Recipes { get; set; } = new(); + } + + [MessagePackObject] + public sealed class IdRecipe + { + [Key(0)] public string Name { get; set; } = ""; + [Key(1)] [CultElementId(nameof(Name))] public string ElementId { get; set; } = ""; + } + [MessagePackObject] public sealed class IdLabel { diff --git a/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs b/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs index ac6a6dad..5ca50c69 100644 --- a/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs +++ b/tests/GameCult.Caching.Tests/EmittedDocumentTypes.cs @@ -12,7 +12,7 @@ namespace GameCult.Caching.Tests // two ways), an alias whose resolved type would depend on scan order, or a type a test needs to itself. internal static class EmittedDocumentTypes { - internal readonly record struct Field(string Name, Type Type, int Key, bool IsName = false); + internal readonly record struct Field(string Name, Type Type, int Key, bool IsName = false, string? Index = null, string? Role = null); internal static Type Emit(string typeName, string schemaName, string schemaVersion, Field[]? fields = null, string[]? compatibleSchemaIds = null) { @@ -37,6 +37,10 @@ namespace GameCult.Caching.Tests builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(KeyAttribute).GetConstructor(new[] { typeof(int) })!, new object[] { field.Key })); if (field.IsName) builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(CultNameAttribute).GetConstructor(Type.EmptyTypes)!, Array.Empty())); + if (field.Index != null) + builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(CultIndexAttribute).GetConstructor(new[] { typeof(string) })!, new object[] { field.Index })); + if (field.Role != null) + builder.SetCustomAttribute(new CustomAttributeBuilder(typeof(CultRoleAttribute).GetConstructor(new[] { typeof(string) })!, new object[] { field.Role })); } return type.CreateType()!; diff --git a/tests/GameCult.Caching.Tests/FirstCallTests.cs b/tests/GameCult.Caching.Tests/FirstCallTests.cs index 8540d9d0..8fad1229 100644 --- a/tests/GameCult.Caching.Tests/FirstCallTests.cs +++ b/tests/GameCult.Caching.Tests/FirstCallTests.cs @@ -14,7 +14,7 @@ using MessagePack; public sealed class Item { [Key(0)] [CultName] public string Name = string.Empty; - [Key(1)] [CultIndex] public string Category = string.Empty; + [Key(1)] public string Category = string.Empty; [Key(2)] public int Value; } diff --git a/tests/GameCult.Caching.Tests/UniqueIndexTests.cs b/tests/GameCult.Caching.Tests/UniqueIndexTests.cs new file mode 100644 index 00000000..45817296 --- /dev/null +++ b/tests/GameCult.Caching.Tests/UniqueIndexTests.cs @@ -0,0 +1,459 @@ +#nullable enable +using System; +using System.IO; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:unique-indexes. Within one document type and one [CultIndex] alias, no two held records hold one + // non-blank value; CultCache.Resolve judges every commit and every load, and a refusal never echoes the value. + public class UniqueIndexTests + { + private const string Canary = "CANARY-7f3a"; + + private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] + { + typeof(UqItem), typeof(UqTwin), typeof(UqTagged), typeof(UqDecimal), typeof(UqDouble), typeof(UqStamp) + }); + + // Emitted, so CultDocumentRegistry.Shared never scans them (one schema declared two ways, and a member Shared would refuse). + // The same schema name, version and shape, once as a role and once as a unique index: a role store holds the pair, the index type refuses it. + private static readonly Type UqLoadRole = EmittedDocumentTypes.Emit( + "UqLoadRole", "tests.uq_load", "tests.uq_load.v1", new[] { new EmittedDocumentTypes.Field("Code", typeof(string), 0, Role: "code") }); + private static readonly Type UqLoadIndex = EmittedDocumentTypes.Emit( + "UqLoadIndex", "tests.uq_load", "tests.uq_load.v1", new[] { new EmittedDocumentTypes.Field("Code", typeof(string), 0, Index: "code") }); + private static readonly Type UqBoth = EmittedDocumentTypes.Emit( + "UqBoth", "tests.uq_both", "tests.uq_both.v1", new[] { new EmittedDocumentTypes.Field("Both", typeof(string), 0, Index: "x", Role: "x") }); + + private static object Load(Type type, string code) => EmittedDocumentTypes.New(type, ("Code", code)); + + private string _directory = string.Empty; + + [SetUp] + public void CreateDirectory() + { + _directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidx-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void DeleteDirectory() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + + private string PathOf(string name) => Path.Combine(_directory, name); + + private static CultCache Open(string path, CultDocumentRegistry? registry = null) => + CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry ?? Registry }); + + private static CultRecordKey Key(string name) => new(name); + + private static UqItem Item(string? sku) => new() { Sku = sku }; + + private static InvalidOperationException Refused(TestDelegate write) => Assert.Throws(write)!; + + [Test] + public void ABatchLeavingTwoRecordsWithOneValueIsRefusedNamingBothKeysAndTheIndex() + { + var path = PathOf("items.cc"); + using var cache = Open(path); + cache.Commit(batch => batch.Upsert(typeof(UqItem), Item("keep"), Key("seed"))); + var before = File.ReadAllBytes(path); + + var refused = Refused(() => cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item(Canary), Key("zeta")); + batch.Upsert(typeof(UqItem), Item(Canary), Key("alpha")); + })); + + Assert.That(refused.Message, Does.Contain("alpha").And.Contain("zeta").And.Contain("sku").And.Contain("tests.uq_item")); + Assert.That(refused.Message.IndexOf("alpha", StringComparison.Ordinal), Is.LessThan(refused.Message.IndexOf("zeta", StringComparison.Ordinal)), "keys in ordinal order"); + Assert.That(refused.Message, Does.Not.Contain(Canary), "the value is never echoed"); + Assert.That(cache.Get(Key("alpha")), Is.Null); + Assert.That(cache.Get(Key("zeta")), Is.Null); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "the store is byte-identical"); + Assert.That(cache.GetByIndex("sku", Canary), Is.Null); + } + + [Test] + public async Task AWriteOfAHeldValueAtAnotherKeyIsRefused() + { + using var cache = Open(PathOf("items.cc")); + await cache.UpsertAsync(typeof(UqItem), Item(Canary), Key("first")); + + var single = Assert.ThrowsAsync(() => cache.UpsertAsync(typeof(UqItem), Item(Canary), Key("second")))!; + Assert.That(single.Message, Does.Contain("first").And.Contain("second").And.Contain("sku").And.Not.Contain(Canary)); + var batched = Refused(() => cache.Commit(batch => batch.Upsert(typeof(UqItem), Item(Canary), Key("third")))); + Assert.That(batched.Message, Does.Contain("first").And.Contain("third")); + + Assert.That(cache.Get(Key("second")), Is.Null); + Assert.That(cache.Get(Key("third")), Is.Null); + Assert.That(cache.GetByIndex("sku", Canary), Is.Not.Null); + + await cache.UpsertAsync(typeof(UqItem), Item(Canary), Key("first")); + } + + [Test] + public void SwappingTwoValuesInOneCommitLandsAndBothAreFound() + { + var path = PathOf("items.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item("one"), Key("a")); + batch.Upsert(typeof(UqItem), Item("two"), Key("b")); + }); + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item("two"), Key("a")); + batch.Upsert(typeof(UqItem), Item("one"), Key("b")); + }); + + Assert.That(cache.GetStored(Key("a"))!.Document, Is.SameAs(cache.GetByIndex("sku", "two"))); + Assert.That(cache.GetStored(Key("b"))!.Document, Is.SameAs(cache.GetByIndex("sku", "one"))); + } + + using var reopened = Open(path); + Assert.That(reopened.GetStored(Key("a"))!.Document, Is.SameAs(reopened.GetByIndex("sku", "two"))); + Assert.That(reopened.GetStored(Key("b"))!.Document, Is.SameAs(reopened.GetByIndex("sku", "one"))); + } + + [Test] + public void AValueMovesWhenItsHolderIsRemovedOrChangedInTheSameCommit() + { + using var cache = Open(PathOf("items.cc")); + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item("one"), Key("a")); + batch.Upsert(typeof(UqItem), Item("two"), Key("b")); + }); + + cache.Commit(batch => + { + batch.Remove(Key("a")); + batch.Upsert(typeof(UqItem), Item("one"), Key("c")); + }); + Assert.That(cache.GetStored(Key("c"))!.Document, Is.SameAs(cache.GetByIndex("sku", "one")), "removed holder released its value"); + + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item("three"), Key("b")); + batch.Upsert(typeof(UqItem), Item("two"), Key("d")); + }); + Assert.That(cache.GetStored(Key("d"))!.Document, Is.SameAs(cache.GetByIndex("sku", "two")), "changed holder released its value"); + Assert.That(cache.GetStored(Key("b"))!.Document, Is.SameAs(cache.GetByIndex("sku", "three"))); + Assert.That(cache.GetByIndex("sku", "one")!.Sku, Is.EqualTo("one")); + } + + [Test] + public void BlankValuesNeverConflict() + { + var path = PathOf("items.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item(null), Key("a")); + batch.Upsert(typeof(UqItem), Item(""), Key("b")); + batch.Upsert(typeof(UqItem), Item(" "), Key("c")); + batch.Upsert(typeof(UqItem), Item(null), Key("d")); + }); + Assert.That(cache.GetByIndex("sku", ""), Is.Null); + Assert.That(cache.GetByIndex("sku", " "), Is.Null); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(Key("d")), Is.Not.Null, "a store of blank values reopens"); + } + + [Test] + public void TwoTypesMayShareAnAliasAndAValue() + { + using var cache = Open(PathOf("items.cc")); + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item("same"), Key("item")); + batch.Upsert(typeof(UqTwin), new UqTwin { Sku = "same" }, Key("twin")); + }); + + Assert.That(cache.GetByIndex("sku", "same"), Is.Not.Null); + Assert.That(cache.GetByIndex("sku", "same"), Is.Not.Null); + Assert.That(Registry.GetRequired().DeclaredMembers.Single().IndexAlias, Is.EqualTo("sku"), "a unique index addresses selection as a role does"); + } + + [Test] + public void RoleMembersMayShareValuesAndStaySelectable() + { + var path = PathOf("tagged.cc"); + using (var cache = Open(path)) + { + cache.Commit(batch => + { + batch.Upsert(typeof(UqTagged), new UqTagged { Kind = "weapon" }, Key("a")); + batch.Upsert(typeof(UqTagged), new UqTagged { Kind = "weapon" }, Key("b")); + }); + Assert.That(cache.GetByIndex("kind", "weapon"), Is.Null, "a role is not a lookup"); + } + + using var reopened = Open(path); + Assert.That(reopened.Get(Key("b")), Is.Not.Null, "a role pair reopens"); + var member = Registry.GetRequired().DeclaredMembers.Single(candidate => candidate.MemberName == nameof(UqTagged.Kind)); + Assert.That(member.IndexAlias, Is.EqualTo("kind"), "the role still supplies the selection address"); + } + + [Test] + public void AStoreHoldingDuplicatesRefusesToOpenNamingBothKeysAndTheIndex() + { + var path = PathOf("load.cc"); + var roleRegistry = CultDocumentRegistry.ForTypes(new[] { UqLoadRole }); + using (var writer = Open(path, roleRegistry)) + { + writer.Commit(batch => + { + batch.Upsert(UqLoadRole, Load(UqLoadRole, Canary), Key("m")); + batch.Upsert(UqLoadRole, Load(UqLoadRole, Canary), Key("k")); + }); + } + + var before = File.ReadAllBytes(path); + var indexRegistry = CultDocumentRegistry.ForTypes(new[] { UqLoadIndex }); + var refused = Refused(() => Open(path, indexRegistry).Dispose()); + + Assert.That(refused.Message, Does.Contain("k").And.Contain("m").And.Contain("code").And.Not.Contain(Canary)); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "the file is untouched"); + + using var cache = new CultCache(indexRegistry); + var store = new SingleFileMessagePackBackingStore(path); + Refused(() => cache.AddBackingStore(store)); + Assert.That(cache.BackingStores, Is.Empty, "the refused store is not attached"); + Assert.That(cache.Get(Key("k")), Is.Null); + } + + [Test] + public void AWriteOntoAMovedFileIsAConflictAndTheRetryIsRefused() + { + var path = PathOf("items.cc"); + using var seed = Open(path); + seed.Commit(batch => batch.Upsert(typeof(UqItem), Item("base"), Key("a"))); + using var second = Open(path); + using (var first = Open(path)) + first.Commit(batch => batch.Upsert(typeof(UqItem), Item(Canary), Key("b"))); + + var conflict = Assert.Throws( + () => second.Commit(batch => batch.Upsert(typeof(UqItem), Item(Canary), Key("c"))))!; + Assert.That(conflict.Message, Does.Contain("unique index")); + Assert.That(second.Get(Key("b")), Is.Not.Null, "the moved record is loaded"); + + var retry = Refused(() => second.Commit(batch => batch.Upsert(typeof(UqItem), Item(Canary), Key("c")))); + Assert.That(retry.Message, Does.Contain("b").And.Contain("c").And.Not.Contain(Canary)); + + using var reopened = Open(path); + Assert.That(reopened.Get(Key("c")), Is.Null, "the file never holds the pair"); + Assert.That(reopened.Get(Key("b")), Is.Not.Null); + } + + [Test] + public void CachelessWritesOfAUniqueIndexedTypeAreRefused() + { + var path = PathOf("items.cc"); + using (var cache = Open(path)) + cache.Commit(batch => batch.Upsert(typeof(UqItem), Item("a"), Key("a"))); + var before = File.ReadAllBytes(path); + var store = new SingleFileMessagePackBackingStore(path); + + Refused(() => store.WriteDocument(Key("x"), Item(Canary), registry: Registry)); + var entry = Registry.GetRequired().ToCatalogEntry(); + Refused(() => store.WriteRecord( + new CultPersistedRecord { Key = "y", SchemaId = entry.SchemaId, StoredAt = DateTimeOffset.UtcNow.ToString("O"), Payload = new byte[] { 0x90 } }, + entry, + registry: Registry)); + + Refused(() => store.WriteRecord( + new CultPersistedRecord { Key = "y", SchemaId = entry.SchemaId, StoredAt = DateTimeOffset.UtcNow.ToString("O"), Payload = new byte[] { 0x90 } }, + entry)); + Refused(() => store.WriteRecord( + new CultPersistedRecord { Key = "y", SchemaId = entry.SchemaId, StoredAt = DateTimeOffset.UtcNow.ToString("O"), Payload = new byte[] { 0x90 } }, + entry, + registry: CultDocumentRegistry.ForTypes(new[] { typeof(UqTagged) }))); + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "the file is byte-identical, whichever registry (none, another) was lent"); + store.WriteDocument(Key("t"), new UqTagged { Kind = "k" }, registry: Registry); + using var reopened = Open(path); + Assert.That(reopened.Get(Key("t")), Is.Not.Null, "a role-only type still writes cacheless"); + } + + + // The rule is a function of the whole commit: it must not stop at a prefix, a mixed batch or a small load. + [TestCase(0, 1)] + [TestCase(0, 6)] + [TestCase(2, 5)] + [TestCase(5, 6)] + public void ADuplicateAnywhereInALargeMixedBatchIsRefused(int first, int second) + { + using var cache = Open(PathOf("items.cc")); + + var refused = Refused(() => cache.Commit(batch => + { + batch.Upsert(typeof(UqTagged), new UqTagged { Kind = "same" }, Key("tag-a")); + for (var i = 0; i < 7; i++) + batch.Upsert(typeof(UqItem), Item(i == first || i == second ? "dup" : "v" + i), Key("item-" + i)); + batch.Upsert(typeof(UqTagged), new UqTagged { Kind = "same" }, Key("tag-b")); + })); + + Assert.That(refused.Message, Does.Contain("item-" + first).And.Contain("item-" + second).And.Not.Contain("dup")); + Assert.That(cache.Get(Key("tag-a")), Is.Null, "nothing landed"); + } + + [Test] + public void ADuplicateBesideARecordOfAnotherTypeSharingTheAliasIsStillRefused() + { + using var cache = Open(PathOf("items.cc")); + var refused = Refused(() => cache.Commit(batch => + { + batch.Upsert(typeof(UqTwin), new UqTwin { Sku = "same" }, Key("twin")); + batch.Upsert(typeof(UqItem), Item("same"), Key("item-1")); + batch.Upsert(typeof(UqItem), Item("same"), Key("item-2")); + })); + Assert.That(refused.Message, Does.Contain("item-1").And.Contain("item-2").And.Not.Contain("twin")); + } + + [Test] + public void ValuesDifferingOnlyByCaseAreDifferentValues() + { + using var cache = Open(PathOf("items.cc")); + cache.Commit(batch => + { + batch.Upsert(typeof(UqItem), Item("Sku"), Key("a")); + batch.Upsert(typeof(UqItem), Item("sku"), Key("b")); + }); + Assert.That(cache.GetStored(Key("a"))!.Document, Is.SameAs(cache.GetByIndex("sku", "Sku"))); + Assert.That(cache.GetStored(Key("b"))!.Document, Is.SameAs(cache.GetByIndex("sku", "sku"))); + } + + [Test] + public void ALoadOfManyRecordsRefusesAPairWhereverItSits([Values(0, 3, 6)] int pairAt) + { + var path = PathOf("load.cc"); + var roleRegistry = CultDocumentRegistry.ForTypes(new[] { UqLoadRole }); + using (var writer = Open(path, roleRegistry)) + { + writer.Commit(batch => + { + for (var i = 0; i < 8; i++) + batch.Upsert(UqLoadRole, Load(UqLoadRole, i == pairAt || i == pairAt + 1 ? "dup" : "v" + i), Key("rec-" + i)); + }); + } + + var refused = Refused(() => Open(path, CultDocumentRegistry.ForTypes(new[] { UqLoadIndex })).Dispose()); + Assert.That(refused.Message, Does.Contain("rec-" + pairAt).And.Contain("rec-" + (pairAt + 1)).And.Not.Contain("dup")); + } + + [Test] + public void AMixedBatchOntoAMovedFileIsAConflictWhateverItsOrder() + { + var path = PathOf("items.cc"); + using var seed = Open(path); + seed.Commit(batch => batch.Upsert(typeof(UqItem), Item("base"), Key("a"))); + using var second = Open(path); + using (var first = Open(path)) + first.Commit(batch => batch.Upsert(typeof(UqItem), Item("moved"), Key("b"))); + + Assert.Throws(() => second.Commit(batch => + { + batch.Upsert(typeof(UqTagged), new UqTagged { Kind = "k" }, Key("tag")); + batch.Upsert(typeof(UqItem), Item("fresh"), Key("c")); + })); + } + + [Test] + public void ANumericIndexValueIsOneCanonicalValueInEveryCulture() + { + var culture = (CultureInfo)CultureInfo.InvariantCulture.Clone(); + culture.NumberFormat.NumberDecimalSeparator = ","; + var previous = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = culture; + try + { + using var cache = Open(PathOf("numbers.cc")); + cache.Commit(batch => batch.Upsert(typeof(UqDecimal), new UqDecimal { Price = 1.0m }, Key("d1"))); + Assert.That(cache.GetByIndex("price", "1"), Is.Not.Null, "1.0m is the canonical value 1"); + var sameDecimal = Refused(() => cache.Commit(batch => batch.Upsert(typeof(UqDecimal), new UqDecimal { Price = 1.00m }, Key("d2")))); + Assert.That(sameDecimal.Message, Does.Contain("d1").And.Contain("d2")); + + cache.Commit(batch => batch.Upsert(typeof(UqDecimal), new UqDecimal { Price = 1.5m }, Key("d3"))); + Assert.That(cache.GetByIndex("price", "1.5"), Is.Not.Null, "lookups name the invariant spelling, not the current culture's"); + Assert.That(cache.GetByIndex("price", "1,5"), Is.Null); + + culture.DateTimeFormat.ShortDatePattern = "yyyy.dd.MM"; + culture.DateTimeFormat.DateSeparator = "."; + cache.Commit(batch => batch.Upsert(typeof(UqStamp), new UqStamp { When = new DateTime(2026, 1, 2, 3, 4, 5) }, Key("t1"))); + Assert.That(cache.GetByIndex("when", "01/02/2026 03:04:05"), Is.Not.Null, "a date is its invariant spelling"); + + cache.Commit(batch => batch.Upsert(typeof(UqDouble), new UqDouble { Level = 0.0 }, Key("z1"))); + var signedZero = Refused(() => cache.Commit(batch => batch.Upsert(typeof(UqDouble), new UqDouble { Level = -0.0 }, Key("z2")))); + Assert.That(signedZero.Message, Does.Contain("z1").And.Contain("z2")); + } + finally + { + CultureInfo.CurrentCulture = previous; + } + } + + [Test] + public void AMemberWithBothAttributesIsRefusedAtRegistration() + { + var refused = Refused(() => CultDocumentRegistry.ForTypes(new[] { UqBoth })); + Assert.That(refused.Message, Does.Contain("Both").And.Contain("[CultIndex]").And.Contain("[CultRole]")); + } + + [CultDocument("tests.uq_item", "tests.uq_item.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class UqItem + { + [Key(0)] [CultIndex("sku")] public string? Sku { get; set; } + } + + [CultDocument("tests.uq_twin", "tests.uq_twin.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class UqTwin + { + [Key(0)] [CultIndex("sku")] public string? Sku { get; set; } + } + + [CultDocument("tests.uq_decimal", "tests.uq_decimal.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class UqDecimal + { + [Key(0)] [CultIndex("price")] public decimal Price { get; set; } + } + + [CultDocument("tests.uq_stamp", "tests.uq_stamp.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class UqStamp + { + [Key(0)] [CultIndex("when")] public DateTime When { get; set; } + } + + [CultDocument("tests.uq_double", "tests.uq_double.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class UqDouble + { + [Key(0)] [CultIndex("level")] public double Level { get; set; } + } + + [CultDocument("tests.uq_tagged", "tests.uq_tagged.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class UqTagged + { + [Key(0)] [CultRole("kind")] public string? Kind { get; set; } + } + } +} diff --git a/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs new file mode 100644 index 00000000..1fd026ae --- /dev/null +++ b/tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs @@ -0,0 +1,648 @@ +#nullable enable +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using GameCult.Caching; +using GameCult.Caching.MessagePack; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Caching.Tests +{ + // variants:ruling:unique-indexes. The canonical index text and the unique-index load verdicts in tests/vectors/unique-index. + // The C# reference is the only writer (CULTCACHE_WRITE_VECTORS=1); every other run reads the committed bytes and + // compares them byte for byte. Other runtimes are held to the same text and the same verdicts. + public class UniqueIndexVectorTests + { + private const string WriteVariable = "CULTCACHE_WRITE_VECTORS"; + private static readonly DateTimeOffset StoredAt = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static bool Writing => Environment.GetEnvironmentVariable(WriteVariable) == "1"; + + private static string VectorDirectory() + { + for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent) + { + var vectors = Path.Combine(dir.FullName, "tests", "vectors"); + if (Directory.Exists(vectors)) return Path.Combine(vectors, "unique-index"); + } + + throw new DirectoryNotFoundException($"tests/vectors not found above {TestContext.CurrentContext.TestDirectory}."); + } + + private static string VectorPath(string name) => Path.Combine(VectorDirectory(), name); + + private static void Check(string name, byte[] reference) + { + var path = VectorPath(name); + if (Writing) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, reference); + return; + } + + Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); + Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}"); + } + + private static byte[] Committed(string name) + { + var path = VectorPath(name); + Assert.That(File.Exists(path), Is.True, $"vector {name} is committed"); + return File.ReadAllBytes(path); + } + + // ---- values.msgpack: [label, kind, value, text-or-nil] ---- + + private sealed record Row(string Label, string Kind, object Value, string? Text); + + private static readonly MethodInfo RenderCanonicalNumber = + typeof(CultDocumentRegistry).GetMethod("RenderCanonicalNumber", BindingFlags.NonPublic | BindingFlags.Static)!; + + private static object? ByIndex(CultCache cache, Type type, string alias, string value) => + typeof(CultCache).GetMethod(nameof(CultCache.GetByIndex))!.MakeGenericMethod(type).Invoke(cache, new object[] { alias, value }); + + private static string? Number(object raw) => (string?)RenderCanonicalNumber.Invoke(null, new[] { raw }); + + // Strings and booleans carry the text the rule states; numbers carry whatever the reference renders, never hand-typed. + private static List Rows() => new() + { + new("plain", "string", "S-1", "S-1"), + new("empty", "string", "", null), + new("space", "string", " ", null), + new("nel", "string", "\u0085", null), + new("nbsp", "string", "\u00A0", null), + new("ideographic-space", "string", "\u3000", null), + new("vertical-tab", "string", "\u000B", null), + new("bom", "string", "\uFEFF", "\uFEFF"), + new("unit-separator", "string", "\u001F", "\u001F"), + new("padded", "string", " a ", " a "), + new("int64-max", "int64", long.MaxValue, Number(long.MaxValue)), + new("int64-min", "int64", long.MinValue, Number(long.MinValue)), + new("uint64-max", "uint64", ulong.MaxValue, Number(ulong.MaxValue)), + new("double-zero", "double", 0.0, Number(0.0)), + new("double-negative-zero", "double", -0.0, Number(-0.0)), + new("double-tenth", "double", 0.1, Number(0.1)), + new("double-1e21", "double", 1e21, Number(1e21)), + new("double-2.5e-7", "double", 2.5e-7, Number(2.5e-7)), + new("double-nan", "double", double.NaN, Number(double.NaN)), + new("double-infinity", "double", double.PositiveInfinity, Number(double.PositiveInfinity)), + new("bool-true", "bool", true, "True"), + new("bool-false", "bool", false, "False"), + }; + + private static byte[] ValuesFile() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + var rows = Rows(); + writer.WriteArrayHeader(rows.Count); + foreach (var row in rows) + { + writer.WriteArrayHeader(4); + writer.Write(row.Label); + writer.Write(row.Kind); + switch (row.Value) + { + case string s: writer.Write(s); break; + case long l: writer.Write(l); break; + case ulong u: writer.Write(u); break; + case double d: writer.Write(d); break; + case bool b: writer.Write(b); break; + default: throw new InvalidOperationException(row.Kind); + } + + if (row.Text == null) writer.WriteNil(); else writer.Write(row.Text); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + private static readonly Dictionary KindTypes = new[] + { + ("string", typeof(string)), ("int64", typeof(long)), ("uint64", typeof(ulong)), ("double", typeof(double)), ("bool", typeof(bool)) + }.ToDictionary(pair => pair.Item1, pair => EmittedDocumentTypes.Emit( + "UqKind" + pair.Item1, "tests.uq_kind_" + pair.Item1, "tests.uq_kind_" + pair.Item1 + ".v1", + new[] { new EmittedDocumentTypes.Field("Value", pair.Item2, 0, Index: "v") })); + + [Test] + public void ValuesFileIsTheReferencesCanonicalText() + { + Check("values.msgpack", ValuesFile()); + + var reader = new MessagePackReader(Committed("values.msgpack")); + var count = reader.ReadArrayHeader(); + var rows = Rows(); + Assert.That(count, Is.EqualTo(rows.Count)); + foreach (var row in rows) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4)); + Assert.That(reader.ReadString(), Is.EqualTo(row.Label)); + Assert.That(reader.ReadString(), Is.EqualTo(row.Kind)); + reader.Skip(); + var text = reader.TryReadNil() ? null : reader.ReadString(); + Assert.That(text, Is.EqualTo(row.Text), row.Label); + + var type = KindTypes[row.Kind]; + var registry = CultDocumentRegistry.ForTypes(new[] { type }); + using var cache = CultCacheMessagePack.Create(Path.Combine(Path.GetTempPath(), $"cultlib-uidxv-{Guid.NewGuid():N}.cc"), new CultCacheOpenOptions { Registry = registry }); + cache.Commit(batch => + { + batch.Upsert(type, EmittedDocumentTypes.New(type, ("Value", row.Value)), new CultRecordKey("one")); + batch.Upsert(type, EmittedDocumentTypes.New(type, ("Value", row.Value)), new CultRecordKey(text == null ? "two" : "one")); + }); + if (text == null) + { + Assert.That(cache.Get(new CultRecordKey("two")), Is.Not.Null, $"{row.Label}: a blank value commits twice"); + foreach (var probe in new[] { "", " ", "NaN", "0", "Infinity", "∞" }) + Assert.That(ByIndex(cache, type, "v", probe), Is.Null, $"{row.Label}: found by '{probe}'"); + } + else + { + Assert.That(ByIndex(cache, type, "v", text), Is.Not.Null, $"{row.Label}: found by its text"); + } + } + } + + // ---- the stores: written through a role, judged by an index ---- + + private static readonly EmittedDocumentTypes.Field[] RoleFields = + { + new("Sku", typeof(string), 0, Role: "Sku"), new("Category", typeof(string), 1) + }; + + private static readonly EmittedDocumentTypes.Field[] IndexFields = + { + new("Sku", typeof(string), 0, Index: "Sku"), new("Category", typeof(string), 1) + }; + + private static readonly Type ItemRole = EmittedDocumentTypes.Emit("UqVecItemRole", "tests.unique_item", "tests.unique_item.v1", RoleFields); + private static readonly Type ItemIndex = EmittedDocumentTypes.Emit("UqVecItemIndex", "tests.unique_item", "tests.unique_item.v1", IndexFields); + + private static readonly (string Key, string Sku, string Category)[] Items = + { + ("item:a", "S-1", "tools"), ("item:b", "S-1", "tools"), ("item:c", " ", "parts"), ("item:d", "\u0085", "parts") + }; + + private static byte[] StoreBytes(string[] keys) + { + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxs-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + var path = Path.Combine(directory, "store.cc"); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions + { + Registry = CultDocumentRegistry.ForTypes(new[] { ItemRole }), StoreFlushOnDispose = true + })) + { + cache.Clock = () => StoredAt; + foreach (var item in Items.Where(item => keys.Contains(item.Key))) + cache.UpsertAsync(ItemRole, EmittedDocumentTypes.New(ItemRole, ("Sku", item.Sku), ("Category", item.Category)), new CultRecordKey(item.Key)).GetAwaiter().GetResult(); + cache.FlushAsync().GetAwaiter().GetResult(); + } + + return File.ReadAllBytes(path); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + private static readonly (string File, string? Alias, string Outcome, string? K1, string? K2)[] Outcomes = + { + ("duplicate-store.cc", "Sku", "refused", "item:a", "item:b"), + ("duplicate-store.cc", null, "open", null, null), + ("blank-store.cc", "Sku", "open", null, null), + }; + + private static byte[] OutcomesFile() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(Outcomes.Length); + foreach (var row in Outcomes) + { + writer.WriteArrayHeader(5); + writer.Write(row.File); + foreach (var text in new[] { row.Alias, row.Outcome, row.K1, row.K2 }) + if (text == null) writer.WriteNil(); else writer.Write(text); + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + private static string? NextText(ref MessagePackReader reader) => reader.TryReadNil() ? null : reader.ReadString(); + + [Test] + public void StoreVectorsHoldTheirVerdicts() + { + Check("duplicate-store.cc", StoreBytes(new[] { "item:a", "item:b", "item:c", "item:d" })); + Check("blank-store.cc", StoreBytes(new[] { "item:c", "item:d" })); + Check("outcomes.msgpack", OutcomesFile()); + + var reader = new MessagePackReader(Committed("outcomes.msgpack")); + var count = reader.ReadArrayHeader(); + Assert.That(count, Is.EqualTo(Outcomes.Length)); + for (var i = 0; i < count; i++) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(5)); + var file = reader.ReadString()!; + var alias = NextText(ref reader); + var outcome = NextText(ref reader)!; + var k1 = NextText(ref reader); + var k2 = NextText(ref reader); + + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxo-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + var path = Path.Combine(directory, "store.cc"); + File.WriteAllBytes(path, Committed(file)); + var before = File.ReadAllBytes(path); + var type = alias == null ? ItemRole : ItemIndex; + var registry = CultDocumentRegistry.ForTypes(new[] { type }); + if (outcome == "open") + { + using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }); + Assert.That(cache.Get(new CultRecordKey("item:c")), Is.Not.Null, file); + } + else + { + var refused = Assert.Throws(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }).Dispose())!; + Assert.That(refused.Message, Is.EqualTo( + $"Records {k1} and {k2} (tests.unique_item) both hold unique index '{alias}'; an index value names one record.")); + Assert.That(refused.Message, Does.Not.Contain("S-1")); + } + + Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{file} is untouched"); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + } + + // ---- refusals.msgpack: [label, ops, refusal]; the pair a refusal names is a function of the resulting state ---- + + private sealed record Schema(string Name, Type Role, Type Index, string SecondField); + + private static Schema Plain(string suffix, string name) => new( + name, + EmittedDocumentTypes.Emit("UqVec" + suffix + "Role", name, name + ".v1", RoleFields), + EmittedDocumentTypes.Emit("UqVec" + suffix + "Index", name, name + ".v1", IndexFields), + "Category"); + + private static readonly Schema[] Schemas = + { + new("tests.unique_item", ItemRole, ItemIndex, "Category"), + new("tests.uq_pair", + EmittedDocumentTypes.Emit("UqVecPairRole", "tests.uq_pair", "tests.uq_pair.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "Alt") }), + EmittedDocumentTypes.Emit("UqVecPairIndex", "tests.uq_pair", "tests.uq_pair.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "Alt") }), + "Alt"), + new("tests.uq_alias", + EmittedDocumentTypes.Emit("UqVecAliasRole", "tests.uq_alias", "tests.uq_alias.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "alt") }), + EmittedDocumentTypes.Emit("UqVecAliasIndex", "tests.uq_alias", "tests.uq_alias.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "alt") }), + "Alt"), + new("tests.uq_wide", + EmittedDocumentTypes.Emit("UqVecWideRole", "tests.uq_wide", "tests.uq_wide.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "\uFF21"), new("Alt", typeof(string), 1, Role: "\U00010400") }), + EmittedDocumentTypes.Emit("UqVecWideIndex", "tests.uq_wide", "tests.uq_wide.v1", + new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "\uFF21"), new("Alt", typeof(string), 1, Index: "\U00010400") }), + "Alt"), + Plain("Alpha", "tests.uq_alpha"), + Plain("Beta", "tests.uq_beta"), + }; + + private static Schema SchemaNamed(string name) => Schemas.Single(schema => schema.Name == name); + + private sealed record Rec(string Key, string Schema, string Sku, string Second = ""); + + private sealed record Op(string Kind, string? File = null, Rec[]? Records = null); + + private sealed record Refusal(string K1, string K2, string Schema, string Alias); + + private sealed record Case(string Label, Op[] Ops, Refusal? Refusal); + + private const string Item = "tests.unique_item"; + private const string Pair = "tests.uq_pair"; + private const string Alias = "tests.uq_alias"; + private const string Wide = "tests.uq_wide"; + private const string Alpha = "tests.uq_alpha"; + private const string Beta = "tests.uq_beta"; + + private static Rec[] Items_(params (string Key, string Sku)[] rows) => rows.Select(row => new Rec(row.Key, Item, row.Sku)).ToArray(); + private static Op Open(string? file = null) => new("open", file); + private static Op Pull(string file) => new("pull", file); + private static Op Commit(params Rec[] records) => new("commit", Records: records); + + private static readonly Rec[] PairRecords = { new("item:a", Pair, "S", "X"), new("item:b", Pair, "S", "Y"), new("item:c", Pair, "T", "Y") }; + private static readonly Rec[] SurrogateRecords = Items_(("k\uFF01", "S"), ("k\U0001F600", "S"), ("k\uE000", "S")); + private static readonly Rec[] TypeRecords = + { + new("b:1", Alpha, "P"), new("a:1", Beta, "P"), new("b:2", Alpha, "P"), new("a:2", Beta, "P") + }; + private static readonly Rec[] AliasRecords = { new("item:a", Alias, "S", "X"), new("item:b", Alias, "S", "Y"), new("item:c", Alias, "T", "Y") }; + private static readonly Rec[] RepullBefore = Items_(("item:a1", "A"), ("item:b1", "C")); + private static readonly Rec[] RepullAfter = Items_(("item:a1", "A"), ("item:b1", "B"), ("item:b2", "B"), ("item:c1", "A")); + private static readonly Rec[] KeyCaseRecords = Items_(("a", "S"), ("z", "S"), ("B", "T"), ("C", "T")); + private static readonly Rec[] HolderCaseRecords = Items_(("a", "S"), ("A", "S")); + private static readonly Rec[] CrossSchemaRecords = { new("x:1", Alpha, "P"), new("y:1", Beta, "P") }; + // Distinct under any trim, case-fold, NFC or NFKC of the value: 'S' and 's', 'S' and 'S ', NFC and NFD, fullwidth S. + private static readonly Rec[] DistinctValueRecords = Items_( + ("k:1", "S"), ("k:2", "s"), ("k:3", "S "), ("k:4", "\u00E9"), ("k:5", "e\u0301"), ("k:6", "\uFF33"), ("k:7", " S")); + private static readonly Rec[] NormalKeyRecords = Items_(("\u00E9", "S"), ("e\u0301", "S")); + // Three holders of one value: code-unit order is e+U+0302, f, U+00E9; NFC would order f, U+00E9, e+U+0302 and NFD U+00E9, e+U+0302, f. + private static readonly Rec[] ThreeHolderRecords = Items_(("e\u0302", "S"), ("f", "S"), ("\u00E9", "S")); + // Both records violate both aliases: by code unit U+10400 (D801 DC00) precedes U+FF21, by code point U+FF21 does. + private static readonly Rec[] WideRecords = { new("item:a", Wide, "S", "X"), new("item:b", Wide, "S", "X") }; + private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T")); + private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T")); + + private static readonly (string File, Rec[] Records)[] RefusalStores = + { + ("pair-store.cc", PairRecords), ("surrogate-store.cc", SurrogateRecords), ("types-store.cc", TypeRecords), + ("repull-before.cc", RepullBefore), ("repull-after.cc", RepullAfter), ("alias-store.cc", AliasRecords), + ("key-case-store.cc", KeyCaseRecords), ("key-punctuation-store.cc", KeyPunctuationRecords), + ("surrogate-groups-store.cc", SurrogateGroupRecords), + ("holder-case-store.cc", HolderCaseRecords), + ("cross-schema-store.cc", CrossSchemaRecords), ("distinct-values-store.cc", DistinctValueRecords), + ("normal-keys-store.cc", NormalKeyRecords), ("normal-three-store.cc", ThreeHolderRecords), ("wide-store.cc", WideRecords), + }; + + private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku"); + private static readonly Refusal TypesRefusal = new("a:1", "a:2", Beta, "Sku"); + private static readonly Refusal PairRefusal = new("item:b", "item:c", Pair, "Alt"); + private static readonly Refusal AliasRefusal = new("item:a", "item:b", Alias, "Sku"); + private static readonly Refusal RepullRefusal = new("item:a1", "item:c1", Item, "Sku"); + private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku"); + private static readonly Refusal HolderCaseRefusal = new("A", "a", Item, "Sku"); + private static readonly Refusal NormalKeyRefusal = new("e\u0301", "\u00E9", Item, "Sku"); + private static readonly Refusal ThreeHolderRefusal = new("e\u0302", "f", Item, "Sku"); + private static readonly Refusal WideRefusal = new("item:a", "item:b", Wide, "\U00010400"); + private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku"); + private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku"); + + private static readonly Case[] Cases = + { + new("batch-later-value-staged-first", new[] { Open(), Commit(Items_(("x", "A"), ("y", "A"), ("a", "B"), ("b", "B"))) }, + new Refusal("a", "b", Item, "Sku")), + new("batch-surrogate-keys", new[] { Open(), Commit(SurrogateRecords) }, SurrogateRefusal), + new("batch-interleaved-types", new[] { Open(), Commit(TypeRecords) }, TypesRefusal), + new("batch-two-aliases", new[] { Open(), Commit(PairRecords) }, PairRefusal), + new("held-then-landing", new[] { Open(), Commit(Items_(("m", "S"))), Commit(Items_(("a", "S"))) }, + new Refusal("a", "m", Item, "Sku")), + new("held-in-the-least-group", new[] { Open(), Commit(Items_(("m", "A"))), Commit(Items_(("z", "B"), ("y", "B"), ("a", "A"))) }, + new Refusal("a", "m", Item, "Sku")), + new("swap-off-a-held-value", new[] { Open(), Commit(Items_(("a1", "A"))), Commit(Items_(("a1", "B"), ("c1", "A"))) }, null), + new("load-two-aliases", new[] { Open("pair-store.cc") }, PairRefusal), + new("load-surrogate-keys", new[] { Open("surrogate-store.cc") }, SurrogateRefusal), + new("load-interleaved-types", new[] { Open("types-store.cc") }, TypesRefusal), + new("repull-held-pair", new[] { Open("repull-before.cc"), Pull("repull-after.cc") }, RepullRefusal), + new("load-repull-after", new[] { Open("repull-after.cc") }, RepullRefusal), + new("commit-onto-a-loaded-holder", new[] { Open("repull-before.cc"), Commit(Items_(("item:z", "A"))) }, + new Refusal("item:a1", "item:z", Item, "Sku")), + // Rows 14-19 extend the spec's floor of 13: orderings a code-point, case-insensitive or culture compare would flip. + new("batch-key-case", new[] { Open(), Commit(KeyCaseRecords) }, KeyCaseRefusal), + new("batch-key-punctuation", new[] { Open(), Commit(KeyPunctuationRecords) }, KeyPunctuationRefusal), + new("batch-key-case-in-a-group", new[] { Open(), Commit(Items_(("a", "S"), ("B", "S"))) }, + new Refusal("B", "a", Item, "Sku")), + new("batch-surrogate-groups", new[] { Open(), Commit(SurrogateGroupRecords) }, SurrogateGroupRefusal), + new("batch-alias-case", new[] { Open(), Commit(AliasRecords) }, AliasRefusal), + new("load-alias-case", new[] { Open("alias-store.cc") }, AliasRefusal), + // Rows 20-22 load rows 14, 15 and 17's records, so a runtime with no batch commit still walks cross-group key order. + new("load-key-case", new[] { Open("key-case-store.cc") }, KeyCaseRefusal), + new("load-key-punctuation", new[] { Open("key-punctuation-store.cc") }, KeyPunctuationRefusal), + new("load-surrogate-groups", new[] { Open("surrogate-groups-store.cc") }, SurrogateGroupRefusal), + // Rows 23-24: two holders of one value that differ only by case are two holders; a case-folding dedupe would see one. + new("batch-holders-by-case", new[] { Open(), Commit(HolderCaseRecords) }, HolderCaseRefusal), + new("load-holders-by-case", new[] { Open("holder-case-store.cc") }, HolderCaseRefusal), + // Row 25: the held `a` is replaced only by the exact key `a`, never by `A`. + new("batch-replaced-by-case", new[] { Open(), Commit(Items_(("a", "S"))), Commit(Items_(("A", "S"))) }, HolderCaseRefusal), + // Rows 26-31: identity is exact. One value in two schemas is two groups; values differing in case, trailing or leading space, normalisation form or width are two values; keys differing only by normalisation form are two holders. + new("batch-value-in-two-schemas", new[] { Open(), Commit(CrossSchemaRecords) }, null), + new("load-value-in-two-schemas", new[] { Open("cross-schema-store.cc") }, null), + new("batch-distinct-values", new[] { Open(), Commit(DistinctValueRecords) }, null), + new("load-distinct-values", new[] { Open("distinct-values-store.cc") }, null), + new("batch-keys-by-normal-form", new[] { Open(), Commit(NormalKeyRecords) }, NormalKeyRefusal), + new("load-keys-by-normal-form", new[] { Open("normal-keys-store.cc") }, NormalKeyRefusal), + // Rows 32-33: one landing record sharing two unique aliases with a held one; the least alias by code unit is named ('Alt' before 'Sku'; 'Sku' before 'alt'). + new("put-onto-a-held-pair", new[] { Open(), Commit(new Rec("m", Pair, "S", "X")), Commit(new Rec("a", Pair, "S", "X")) }, + new Refusal("a", "m", Pair, "Alt")), + new("put-onto-a-held-alias-pair", new[] { Open(), Commit(new Rec("m", Alias, "S", "X")), Commit(new Rec("a", Alias, "S", "X")) }, + new Refusal("a", "m", Alias, "Sku")), + // Rows 34-35: three holders of one value; the two least by code unit are named, which no NFC, NFD or culture sort of the keys yields. + new("batch-three-holders-normal-form", new[] { Open(), Commit(ThreeHolderRecords) }, ThreeHolderRefusal), + new("load-three-holders-normal-form", new[] { Open("normal-three-store.cc") }, ThreeHolderRefusal), + // Rows 36-37: two records violating two aliases, U+FF21 and U+10400; aliases compare by code unit, so U+10400 is named. + new("batch-alias-code-unit", new[] { Open(), Commit(WideRecords) }, WideRefusal), + new("load-alias-code-unit", new[] { Open("wide-store.cc") }, WideRefusal), + }; + + private static byte[] RefusalStoreBytes(Rec[] records) + { + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxr-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + try + { + var path = Path.Combine(directory, "store.cc"); + var roles = records.Select(record => SchemaNamed(record.Schema).Role).Distinct().ToArray(); + using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions + { + Registry = CultDocumentRegistry.ForTypes(roles), StoreFlushOnDispose = true + })) + { + cache.Clock = () => StoredAt; + foreach (var record in records) + { + var schema = SchemaNamed(record.Schema); + cache.UpsertAsync(schema.Role, EmittedDocumentTypes.New(schema.Role, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key)) + .GetAwaiter().GetResult(); + } + + cache.FlushAsync().GetAwaiter().GetResult(); + } + + return File.ReadAllBytes(path); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + private static byte[] RefusalsFile() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(Cases.Length); + foreach (var row in Cases) + { + writer.WriteArrayHeader(3); + writer.Write(row.Label); + writer.WriteArrayHeader(row.Ops.Length); + foreach (var op in row.Ops) + { + writer.WriteArrayHeader(2); + writer.Write(op.Kind); + if (op.Records != null) + { + writer.WriteArrayHeader(op.Records.Length); + foreach (var record in op.Records) + { + writer.WriteArrayHeader(4); + writer.Write(record.Key); + writer.Write(record.Schema); + writer.Write(record.Sku); + writer.Write(record.Second); + } + } + else if (op.File == null) writer.WriteNil(); + else writer.Write(op.File); + } + + if (row.Refusal == null) writer.WriteNil(); + else + { + writer.WriteArrayHeader(4); + writer.Write(row.Refusal.K1); + writer.Write(row.Refusal.K2); + writer.Write(row.Refusal.Schema); + writer.Write(row.Refusal.Alias); + } + } + + writer.Flush(); + return buffer.WrittenSpan.ToArray(); + } + + // Performs one op of a row against the open cache; the cache's own file is the only state it leaves behind. + private static void Perform(ref CultCache? cache, string path, Op op) + { + var indexTypes = Schemas.Select(schema => schema.Index).ToArray(); + switch (op.Kind) + { + case "open": + if (op.File != null) File.WriteAllBytes(path, Committed(op.File)); + cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(indexTypes) }); + break; + case "pull": + File.WriteAllBytes(path, Committed(op.File!)); + cache!.PullAllBackingStoresAsync().GetAwaiter().GetResult(); + break; + case "commit": + cache!.Commit(batch => + { + foreach (var record in op.Records!) + { + var schema = SchemaNamed(record.Schema); + batch.Upsert(schema.Index, EmittedDocumentTypes.New(schema.Index, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key)); + } + }); + break; + default: + throw new InvalidOperationException(op.Kind); + } + } + + [Test] + public void RefusalsHoldTheRule() + { + Check("refusals.msgpack", RefusalsFile()); + foreach (var (file, records) in RefusalStores) + Check(file, RefusalStoreBytes(records)); + + // The committed file is what is walked, not the in-memory table. + var reader = new MessagePackReader(Committed("refusals.msgpack")); + var count = reader.ReadArrayHeader(); + Assert.That(count, Is.EqualTo(Cases.Length)); + var walked = 0; + var failed = new List(); + for (var i = 0; i < count; i++) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(3)); + var label = reader.ReadString()!; + var ops = new List(); + for (var n = reader.ReadArrayHeader(); n > 0; n--) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2)); + var kind = reader.ReadString()!; + if (kind == "commit") + { + var records = new List(); + for (var m = reader.ReadArrayHeader(); m > 0; m--) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4)); + records.Add(new Rec(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!)); + } + + ops.Add(new Op(kind, Records: records.ToArray())); + } + else ops.Add(new Op(kind, NextText(ref reader))); + } + + Refusal? refusal = null; + if (!reader.TryReadNil()) + { + Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4)); + refusal = new Refusal(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!); + } + + var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxf-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + CultCache? cache = null; + try + { + var path = Path.Combine(directory, "store.cc"); + foreach (var op in ops.Take(ops.Count - 1)) + Perform(ref cache, path, op); + + var last = ops[^1]; + if (refusal == null) + { + Assert.DoesNotThrow(() => Perform(ref cache, path, last), label); + } + else + { + cache?.FlushAsync().GetAwaiter().GetResult(); + var before = last.Kind == "open" && last.File != null ? Committed(last.File) + : last.Kind == "pull" ? Committed(last.File!) + : File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty(); + CultCache? attempt = cache; + var refused = Assert.Throws(() => Perform(ref attempt, path, last), label)!; + Assert.That(refused.Message, Is.EqualTo( + $"Records {refusal.K1} and {refusal.K2} ({refusal.Schema}) both hold unique index '{refusal.Alias}'; an index value names one record."), label); + if (last.Kind != "open") cache?.FlushAsync().GetAwaiter().GetResult(); + Assert.That(File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty(), Is.EqualTo(before), $"{label}: the file is untouched"); + } + + walked++; + } + catch (AssertionException failure) + { + failed.Add($"{label}: {failure.Message}"); + } + finally + { + cache?.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + + Assert.That(failed, Is.Empty, "rows that failed"); + Assert.That(walked, Is.EqualTo(37)); + } + } +} diff --git a/tests/GameCult.Caching.Tests/VariantTests.cs b/tests/GameCult.Caching.Tests/VariantTests.cs index 8f574cd1..17f9ba26 100644 --- a/tests/GameCult.Caching.Tests/VariantTests.cs +++ b/tests/GameCult.Caching.Tests/VariantTests.cs @@ -580,7 +580,7 @@ namespace GameCult.Caching.Tests { cache.Override(nameof(VariantGear.Name), "laser big") }))); - Assert.That(inherited.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value).And.Contain("code").And.Contain("l1")); + Assert.That(inherited.Message, Does.Contain(BigKey.Value).And.Contain(BaseKey.Value).And.Contain("code").And.Not.Contain("l1"), "the value is not echoed"); var explicitly = Refused(() => cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] { @@ -631,7 +631,7 @@ namespace GameCult.Caching.Tests SeedBig(cache); var refused = Refused(() => cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("laser big"), other))); - Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Contain("laser big")); + Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Not.Contain("laser big")); var single = Assert.ThrowsAsync( () => cache.UpsertAsync(Other("laser big"), new CultRecordHandle(other)))!; @@ -660,7 +660,7 @@ namespace GameCult.Caching.Tests cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("laser big"), other)); var refused = Refused(() => SeedBig(cache)); - Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain(other.Value).And.Contain("code").And.Contain("laser big")); + Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain(other.Value).And.Contain("code").And.Not.Contain("laser big")); Assert.That(cache.Get(BigKey), Is.Null, "nothing landed"); // The same batch, either staging order: the result is what is judged. @@ -685,8 +685,9 @@ namespace GameCult.Caching.Tests })); Assert.That(sameBatchVariantFirst.Message, Does.Contain(BigKey.Value).And.Contain("taker")); - // Plain-vs-plain sharing is not this rule's business. - Assert.DoesNotThrow(() => cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("l1"), new CultRecordKey("twin")))); + // Plain-vs-plain sharing is the same rule. + var plain = Refused(() => cache.Commit(batch => batch.Upsert(typeof(VariantGear), Other("l1"), new CultRecordKey("twin")))); + Assert.That(plain.Message, Does.Contain("twin").And.Contain(BaseKey.Value)); } using var reopened = Open(path); @@ -819,7 +820,7 @@ namespace GameCult.Caching.Tests using var a = Open(path); using (var b = Open(path)) b.Commit(batch => batch.Upsert(typeof(VariantGear), Other("zc"), z)); - a.Commit(batch => batch.Upsert(typeof(VariantGear), Other("yc"), new CultRecordKey("y"))); + a.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "yc" }, new CultRecordKey("y"))); var conflict = Assert.Throws(() => SeedBig(a))!; @@ -1100,22 +1101,8 @@ namespace GameCult.Caching.Tests Assert.That(reloaded.Get(BigKey)!.Tags, Is.EqualTo(new[] { "beam", "pierce" })); } - // ---- the held index is truthful: every holder of a value, so no rule judges a variant against a lossy view ---- - private static VariantGear Named(string name, string code) => new() { Name = name, Power = 1, Code = code }; - private static CultRecordKey[] SharedCodePair(CultCache cache) - { - var p1 = new CultRecordKey("p1"); - var p2 = new CultRecordKey("p2"); - cache.Commit(batch => - { - batch.Upsert(typeof(VariantGear), Named("p1", "x"), p1); - batch.Upsert(typeof(VariantGear), Named("p2", "x"), p2); - }); - return new[] { p1, p2 }; - } - private static void VariantOnCode(CultCache cache, string code) => cache.Commit(batch => batch.UpsertVariant(BigKey, BaseKey, new[] { @@ -1123,73 +1110,6 @@ namespace GameCult.Caching.Tests cache.Override(nameof(VariantGear.Code), code) })); - [Test] - public void RemovingTheHolderADuplicateValueFindsLeavesTheOtherFindable() - { - var path = PathOf("gear.cc"); - using (var cache = Open(path)) - { - var pair = SharedCodePair(cache); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1"), "the smallest key"); - - cache.Remove(pair[0]); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p2"), "removing the holder finds the other one"); - - cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("p1", "x"), pair[0])); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1")); - cache.Remove(pair[1]); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("p1"), "removing the other changes nothing"); - cache.Remove(pair[0]); - Assert.That(cache.GetByIndex("code", "x"), Is.Null); - cache.BackingStores[0].PushAll(); - } - - using var reopened = Open(path); - Assert.That(reopened.GetByIndex("code", "x"), Is.Null); - } - - [Test] - public void AVariantOnAValueAPlainRecordStillHoldsAfterItsTwinWasRemovedIsRefusedAndTheStoreReopens() - { - var path = PathOf("gear.cc"); - using (var cache = Open(path)) - { - SeedBase(cache); - var pair = SharedCodePair(cache); - cache.Remove(pair[1]); - - var refused = Refused(() => VariantOnCode(cache, "x")); - Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain("p1").And.Contain("code").And.Contain("'x'")); - Assert.That(cache.Get(BigKey), Is.Null, "nothing landed"); - cache.BackingStores[0].PushAll(); - } - - using var reopened = Open(path); - Assert.That(reopened.Get(BigKey), Is.Null); - Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("p1")); - } - - [Test] - public void AVariantOnAValueAPlainRecordStillHoldsAfterItsTwinChangedItsValueIsRefusedAndTheStoreReopens() - { - var path = PathOf("gear.cc"); - using (var cache = Open(path)) - { - SeedBase(cache); - var pair = SharedCodePair(cache); - cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("p2", "y"), pair[1])); - - var refused = Refused(() => VariantOnCode(cache, "x")); - Assert.That(refused.Message, Does.Contain(BigKey.Value).And.Contain("p1").And.Contain("'x'")); - Assert.DoesNotThrow(() => VariantOnCode(cache, "z"), "an unshared value is fine"); - cache.BackingStores[0].PushAll(); - } - - using var reopened = Open(path); - Assert.That(reopened.GetByIndex("code", "z")!.Name, Is.EqualTo("laser big")); - Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("p1")); - } - // ---- a merge onto the file is judged on the set the file will hold ---- [Test] @@ -1218,7 +1138,7 @@ namespace GameCult.Caching.Tests var conflict = Assert.Throws(Write)!; Assert.That(conflict.ChangedKeys, Does.Contain(BigKey.Value)); var refused = Refused(Write); - Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Contain("shared"), "the retry is judged by Resolve"); + Assert.That(refused.Message, Does.Contain(other.Value).And.Contain(BigKey.Value).And.Contain("code").And.Not.Contain("shared"), "the retry is judged by Resolve"); Assert.That(b.Get(other), Is.Null, "nothing landed in memory"); } @@ -1408,24 +1328,24 @@ namespace GameCult.Caching.Tests { var path = PathOf("gear.cc"); var other = new CultRecordKey("other"); - var narrow = CultDocumentRegistry.ForTypes(new[] { typeof(VariantGear) }); + var narrow = CultDocumentRegistry.ForTypes(new[] { typeof(VariantOther) }); using (var seed = Open(path)) SeedBase(seed); using (var a = Open(path)) using (var b = Open(path, registry: narrow)) { - a.Commit(batch => batch.Upsert(typeof(VariantOther), new VariantOther { Name = "foreign" }, new CultRecordKey("foreign"))); + a.Commit(batch => batch.Upsert(typeof(VariantGear), Named("foreign", "foreign-code"), new CultRecordKey("foreign"))); Assert.That(b.Commit(batch => { batch.Expect(other, null); - batch.Upsert(typeof(VariantGear), Named("other", "fine"), other); + batch.Upsert(typeof(VariantOther), new VariantOther { Name = "fine" }, other); }), Is.True); } using var reopened = Open(path); - Assert.That(reopened.Get(other), Is.Not.Null); - Assert.That(reopened.Get(new CultRecordKey("foreign")), Is.Not.Null, "the other writer's record survives the merge"); + Assert.That(reopened.Get(other), Is.Not.Null); + Assert.That(reopened.Get(new CultRecordKey("foreign")), Is.Not.Null, "the other writer's record survives the merge"); } [Test] @@ -1448,28 +1368,6 @@ namespace GameCult.Caching.Tests } } - [Test] - public void PlainDuplicatesFindTheSmallestKeyLiveAfterAFlushAndAfterAReopen() - { - var path = PathOf("gear.cc"); - using (var cache = Open(path)) - { - cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("nb", "x"), new CultRecordKey("b"))); - cache.Commit(batch => batch.Upsert(typeof(VariantGear), Named("na", "x"), new CultRecordKey("a"))); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "live"); - cache.BackingStores[0].PushAll(); - Assert.That(cache.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "after a flush"); - } - - using (var reopened = Open(path)) - { - Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "after a reopen"); - reopened.Remove(new CultRecordKey("a")); - reopened.Commit(batch => batch.Upsert(typeof(VariantGear), Named("na", "x"), new CultRecordKey("a"))); - Assert.That(reopened.GetByIndex("code", "x")!.Name, Is.EqualTo("na"), "rewriting the smaller key does not change the choice"); - } - } - [Test] public void ADroppedSlotIsReportedOncePerRecord() { diff --git a/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs b/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs new file mode 100644 index 00000000..b38592ec --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs @@ -0,0 +1,849 @@ +#nullable enable +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GameCult.Caching; +using LiteNetLib; +using R3; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + // variants:ruling:replica-commit-grouped-changes. The authoritative cache judges a commit on its result; a replica applying the + // changes one record at a time would judge each intermediate state, where a swap of two [CultIndex] values holds two holders. + // The subscription server tags the changes of one cache commit, the client applies them as one commit, and a replica that + // refuses a commit says so. No transport: the server and client are joined in memory, and every message crosses MessagePack. + public sealed class CultNetReplicaCommitTests + { + [CultDocument("tests.replica_commit_item", "tests.replica_commit_item.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class ReplicaItem + { + [Key(0)] + [CultIndex("sku")] + public string Sku = string.Empty; + + [Key(1)] + public string Note = string.Empty; + } + + // The change message as a peer from before commit identity knew it. + [MessagePackObject] + public sealed class LegacyChange + { + [Key("schemaVersion")] public string SchemaVersion { get; set; } = string.Empty; + [Key("messageId")] public string MessageId { get; set; } = string.Empty; + [Key("subscriptionId")] public string SubscriptionId { get; set; } = string.Empty; + [Key("changeKind")] public string ChangeKind { get; set; } = string.Empty; + [Key("document")] public CultNetRawDocumentRecord? Document { get; set; } + [Key("recordKey")] public string? RecordKey { get; set; } + [Key("schemaId")] public string? SchemaId { get; set; } + } + + private sealed class LoopbackServer : ICultNetSchemaServer + { + private readonly Dictionary _handlers = new(); + public void OnCultNet(Func callback) where T : ICultNetSchemaMessage => _handlers[typeof(T)] = callback; + public void RemoveCultNetMessageListener(Delegate callback) where T : ICultNetSchemaMessage => _handlers.Remove(typeof(T)); + public Task Dispatch(T message, ICultNetSchemaServerPeer peer) where T : ICultNetSchemaMessage => + ((Func)_handlers[typeof(T)])(message, peer); + } + + private sealed class Client : ICultNetSchemaClient + { + private readonly LoopbackServer _server; + private readonly Dictionary> _handlers = new(); + private readonly Peer _peer; + public Client(LoopbackServer server, Peer peer) { _server = server; _peer = peer; peer.Client = this; } + public bool Connected => true; + public void Connect(string host, int port) { } + public void SendCultNet(T message) where T : ICultNetSchemaMessage + { + if (message is CultNetDatabaseSubscribeMessage subscribe) + _server.Dispatch(subscribe, _peer).GetAwaiter().GetResult(); + else if (message is CultNetDatabaseUnsubscribeMessage unsubscribe) + _server.Dispatch(unsubscribe, _peer).GetAwaiter().GetResult(); + } + public void OnCultNet(Action callback) where T : ICultNetSchemaMessage + { + if (!_handlers.TryGetValue(typeof(T), out var list)) _handlers[typeof(T)] = list = new List(); + list.Add(callback); + } + public void Emit(T message) where T : ICultNetSchemaMessage + { + if (_handlers.TryGetValue(typeof(T), out var list)) + foreach (var handler in list.ToArray()) ((Action)handler)(message); + } + public void Dispose() { } + } + + private sealed class Peer : ICultNetSchemaServerPeer + { + public Client? Client; + public ConcurrentQueue Sent { get; } = new(); + public void SendCultNet(T message) where T : ICultNetSchemaMessage + { + // The wire: serialized, then read back as the receiver would. + var bytes = CultNetSchemaMessageSerialization.Serialize(message); + Sent.Enqueue(bytes); + Client!.Emit(CultNetSchemaMessageSerialization.Deserialize(bytes)); + } + public CultNetDatabaseChangeRawMessage[] Changes => Sent + .Select(bytes => CultNetSchemaMessageSerialization.Deserialize(bytes)) + .OfType().ToArray(); + } + + private sealed class Replication : IDisposable + { + public CultCache Source { get; } = new(); + public CultCache Replica { get; } = new(); + public Peer Peer { get; } = new(); + public CultNetDatabaseSubscriptionClient Client { get; } + public List Faults { get; } = new(); + public CultNetDatabase Database { get; } + private readonly CultNetDatabaseSubscriptionServer _server; + public void Reconcile() => _server.Reconcile(); + public object Gate => typeof(CultNetDatabaseSubscriptionServer) + .GetField("_lifecycleGate", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!.GetValue(_server)!; + + public Replication(Action? seedSource = null, Action? seedReplica = null, + Func? authorizeRecord = null) + { + seedSource?.Invoke(Source); + seedReplica?.Invoke(Replica); + Database = new CultNetDatabase(Source); + var wire = new LoopbackServer(); + _server = new CultNetDatabaseSubscriptionServer(wire, Database, authorizeRecord: authorizeRecord); + var documents = new CultNetDocumentRegistry(Replica.Registry) + .Register(CultNetDocumentBinding.ForDocument(Replica.Registry)); + Client = new CultNetDatabaseSubscriptionClient(new Client(wire, Peer), Replica, documents); + Client.Faulted += Faults.Add; + Client.SubscribeAsync("items", schemaIds: new[] { Source.Registry.GetRequired().SchemaId }) + .GetAwaiter().GetResult(); + } + + public ReplicaItem? ReplicaItem(string key) => Replica.Get(new CultRecordKey(key)); + + public void Dispose() + { + Client.Dispose(); + _server.Dispose(); + Database.Dispose(); + } + } + + private static ReplicaItem Item(string sku) => new() { Sku = sku }; + + private static void Stage(CultCache cache, params (string Key, string Sku)[] items) => + Assert.That(cache.Commit(batch => + { + foreach (var (key, sku) in items) + batch.Upsert(typeof(ReplicaItem), Item(sku), new CultRecordKey(key)); + }), Is.True); + + private static async Task Until(Func condition, int seconds = 5) + { + var deadline = DateTime.UtcNow.AddSeconds(seconds); + while (!condition()) + { + if (DateTime.UtcNow > deadline) Assert.Fail("The replica did not reach the expected state."); + await Task.Delay(5); + } + } + + [Test] + public async Task ASwapOfTwoIndexValuesReachesTheReplicaIntact() + { + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two"))); + Assert.That(run.ReplicaItem("a")!.Sku, Is.EqualTo("one")); + + Stage(run.Source, ("a", "two"), ("b", "one")); + await Until(() => run.ReplicaItem("a")!.Sku == "two"); + + Assert.That(run.ReplicaItem("b")!.Sku, Is.EqualTo("one")); + Assert.That(run.Client.Fault, Is.Null); + Assert.That(run.Faults, Is.Empty); + Assert.That(run.Replica.GetByIndex("sku", "two"), Is.SameAs(run.ReplicaItem("a"))); + var swap = run.Peer.Changes; + Assert.That(swap.Select(change => change.CommitId).Distinct().Single(), Is.Not.Null, "the commit is named on the wire"); + Assert.That(swap.Select(change => change.CommitCount), Is.All.EqualTo(2)); + } + + [Test] + public async Task AValueMovedToANewRecordWhileItsOldHolderIsRemovedReachesTheReplica() + { + using var run = new Replication(source => Stage(source, ("a", "one"))); + + Assert.That(run.Source.Commit(batch => + { + batch.Remove(new CultRecordKey("a")); + batch.Upsert(typeof(ReplicaItem), Item("one"), new CultRecordKey("c")); + }), Is.True); + await Until(() => run.ReplicaItem("c") != null); + + Assert.That(run.ReplicaItem("a"), Is.Null); + Assert.That(run.Client.Fault, Is.Null); + Assert.That(run.Replica.GetByIndex("sku", "one"), Is.SameAs(run.ReplicaItem("c"))); + } + + [Test] + public async Task ACommitOfOneChangeCarriesNoCommitFieldsAndIsApplied() + { + using var run = new Replication(); + + Stage(run.Source, ("a", "one")); + await Until(() => run.ReplicaItem("a") != null); + + var change = run.Peer.Changes.Single(); + Assert.That(change.CommitId, Is.Null); + Assert.That(change.CommitCount, Is.Null); + } + + [Test] + public async Task AReplicaThatRefusesACommitFaultsLoudlyAndAppliesNothingMore() + { + // The replica holds a record the source does not, under the value the source now gives to another record: a conflict + // the replica cannot resolve, not an intermediate state. + using var run = new Replication(seedReplica: replica => Stage(replica, ("local", "taken"))); + + Stage(run.Source, ("a", "taken"), ("b", "free")); + await Until(() => run.Client.Fault != null); + + Assert.That(run.Faults, Has.Count.EqualTo(1)); + Assert.That(run.Faults[0], Is.SameAs(run.Client.Fault)); + Assert.That(run.Faults[0].Message, Does.Contain("sku")); + Assert.That(run.ReplicaItem("a"), Is.Null, "the refused commit is not partly applied"); + Assert.That(run.ReplicaItem("b"), Is.Null, "the refused commit is not partly applied"); + + Stage(run.Source, ("c", "later")); + await Task.Delay(100); + Assert.That(run.ReplicaItem("c"), Is.Null, "a replica known to differ takes nothing on top"); + Assert.That(run.Faults, Has.Count.EqualTo(1)); + } + + [Test] + public void ATwoChangeCommitIsWrittenAsTheVectorOtherRuntimesDecode() + { + // tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, as the C# + // serializer writes them, concatenated. Written with CULTCACHE_WRITE_VECTORS=1; every other run compares bytes. + var commit = Enumerable.Range(0, 2).Select(index => new CultNetDatabaseChangeRawMessage + { + MessageId = $"change-{index}", + SubscriptionId = "sub-1", + ChangeKind = "removed", + RecordKey = index == 0 ? "note:a" : "note:b", + SchemaId = "schema-note", + CommitId = "9f1c0f0e4b7a4c1f9d2e", + CommitCount = 2 + }); + var bytes = commit.SelectMany(message => CultNetSchemaMessageSerialization.Serialize(message)).ToArray(); + + var directory = TestContext.CurrentContext.TestDirectory; + while (directory != null && !System.IO.Directory.Exists(System.IO.Path.Combine(directory, "tests", "vectors"))) + directory = System.IO.Path.GetDirectoryName(directory); + Assert.That(directory, Is.Not.Null, "tests/vectors is above the test directory"); + var path = System.IO.Path.Combine(directory!, "tests", "vectors", "database-change-commit", "two-change-commit.msgpack"); + if (Environment.GetEnvironmentVariable("CULTCACHE_WRITE_VECTORS") == "1") + { + System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!); + System.IO.File.WriteAllBytes(path, bytes); + return; + } + + Assert.That(System.IO.File.Exists(path), Is.True, "the vector is committed"); + Assert.That(Convert.ToHexString(bytes), Is.EqualTo(Convert.ToHexString(System.IO.File.ReadAllBytes(path)))); + } + + [Test] + public void AnOlderPeerReadsTheChangeAndIgnoresTheCommitFields() + { + var tagged = new CultNetDatabaseChangeRawMessage + { + MessageId = "m", + SubscriptionId = "items", + ChangeKind = "removed", + RecordKey = "a", + SchemaId = "tests.replica_commit_item.v1", + CommitId = "7", + CommitCount = 2 + }; + + var legacy = MessagePackSerializer.Deserialize( + CultNetSchemaMessageSerialization.Serialize(tagged), CultNetSchemaMessageSerialization.Options); + + Assert.That(legacy.ChangeKind, Is.EqualTo("removed")); + Assert.That(legacy.RecordKey, Is.EqualTo("a")); + Assert.That(legacy.SubscriptionId, Is.EqualTo("items")); + } + + [Test] + public async Task ASwapFromAServerThatDoesNotTagCommitsStillFaultsInsteadOfDiverging() + { + // An older server sends the two halves of a swap untagged: each is judged alone and refused. The refusal is surfaced. + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two"))); + var raw = (string key, string sku) => run.Database.Documents.ToRawRecord( + run.Source.Registry.GetRequired(), new CultRecordKey(key), Item(sku), "2026-01-01T00:00:00Z"); + + run.Peer.Client!.Emit(new CultNetDatabaseChangeRawMessage + { + MessageId = "x", SubscriptionId = "items", ChangeKind = "updated", Document = raw("a", "two") + }); + await Until(() => run.Client.Fault != null); + + Assert.That(run.ReplicaItem("a")!.Sku, Is.EqualTo("one")); + } + + [Test] + public async Task ARotationOfThreeValuesReachesTheReplicaIntact() + { + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two"), ("c", "three"))); + + Stage(run.Source, ("a", "two"), ("b", "three"), ("c", "one")); + await Until(() => run.ReplicaItem("a")!.Sku == "two"); + + Assert.That(new[] { "a", "b", "c" }.Select(key => run.ReplicaItem(key)!.Sku), Is.EqualTo(new[] { "two", "three", "one" })); + Assert.That(run.Client.Fault, Is.Null); + Assert.That(run.Peer.Changes.Select(change => change.CommitCount), Is.All.EqualTo(3)); + } + + // The database server is the other producer of change messages; it names a commit the same way. + [Test] + public void TheDatabaseServerTagsTheChangesOfOneCommitAndNotALoneChange() + { + var cache = new CultCache(); + Stage(cache, ("a", "one"), ("b", "two")); + using var database = new CultNetDatabase(cache); + using var server = new Server(cache, ServerSecurityOptions.Development()); + using var databaseServer = new CultNetDatabaseServer(server, database); + var commits = new List(); + using var watch = database.WatchCommits().Subscribe(commits.Add); + var selection = new CultNetSelection + { + Schemas = new[] { cache.Registry.GetRequired().SchemaId }, + Projection = CultNetSelectionProjections.Document + }; + IReadOnlyList Deliver(CultNetDatabaseCommit commit) => + (IReadOnlyList)typeof(CultNetDatabaseServer) + .GetMethod("DeliverCommit", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)! + .Invoke(databaseServer, new object[] { commit, "sub", selection, new HashSet(), (Action)(_ => { }) })!; + + Stage(cache, ("a", "two"), ("b", "one")); + Stage(cache, ("c", "three")); + + var swap = Deliver(commits[0]); + Assert.That(swap, Has.Count.EqualTo(2)); + Assert.That(swap.Select(change => change.CommitId).Distinct().Single(), Is.EqualTo(commits[0].Id)); + Assert.That(swap.Select(change => change.CommitCount), Is.All.EqualTo(2)); + var lone = Deliver(commits[1]).Single(); + Assert.That(lone.CommitId, Is.Null); + Assert.That(lone.CommitCount, Is.Null); + Assert.That(commits[0].Id, Is.Not.EqualTo(commits[1].Id)); + } + + [Test] + public void ASnapshotThatSwapsIndexValuesAgainstWhatTheReplicaHoldsIsApplied() + { + using var run = new Replication( + source => Stage(source, ("a", "two"), ("b", "one")), + replica => Stage(replica, ("a", "one"), ("b", "two"))); + + Assert.That(run.ReplicaItem("a")!.Sku, Is.EqualTo("two")); + Assert.That(run.ReplicaItem("b")!.Sku, Is.EqualTo("one")); + } + + // Hand-made wire messages: what a peer sends that the real server would not. + private static CultNetDatabaseChangeRawMessage Put(Replication run, string subscription, string key, string sku, string? commit, int count, string? id = null) => + new() + { + MessageId = id ?? Guid.NewGuid().ToString("N"), + SubscriptionId = subscription, + ChangeKind = "updated", + Document = run.Database.Documents.ToRawRecord( + run.Source.Registry.GetRequired(), new CultRecordKey(key), Item(sku), "2026-01-01T00:00:00Z"), + CommitId = commit, + CommitCount = commit == null ? null : count + }; + + private static CultNetDatabaseChangeRawMessage Removal(string subscription, string key, string commit, int count) => + new() + { + MessageId = Guid.NewGuid().ToString("N"), + SubscriptionId = subscription, + ChangeKind = "removed", + RecordKey = key, + SchemaId = "tests.replica_commit_item.v1", + CommitId = commit, + CommitCount = count + }; + + [Test] + public async Task TwoSubscriptionsMayUseOneCommitIdWithoutJoiningTheirCommits() + { + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two"))); + run.Client.SubscribeAsync("other", recordKeys: new[] { "nothing-matches" }).GetAwaiter().GetResult(); + var emit = run.Peer.Client!; + + emit.Emit(Put(run, "items", "a", "two", "9", 2)); + emit.Emit(Put(run, "other", "x", "ex", "9", 2)); + await Task.Delay(100); + Assert.That(run.ReplicaItem("a")!.Sku, Is.EqualTo("one"), "one change of each of two commits is not a commit"); + Assert.That(run.ReplicaItem("x"), Is.Null); + + emit.Emit(Put(run, "items", "b", "one", "9", 2)); + await Until(() => run.ReplicaItem("a")!.Sku == "two"); + Assert.That(run.ReplicaItem("b")!.Sku, Is.EqualTo("one")); + Assert.That(run.ReplicaItem("x"), Is.Null); + Assert.That(run.Client.Fault, Is.Null); + } + + [Test] + public async Task ARemovalAfterTheFirstChangeOfACommitIsApplied() + { + using var run = new Replication(source => Stage(source, ("a", "one"))); + var emit = run.Peer.Client!; + + emit.Emit(Put(run, "items", "c", "one", "5", 2)); + emit.Emit(Removal("items", "a", "5", 2)); + await Until(() => run.ReplicaItem("c") != null); + + Assert.That(run.ReplicaItem("a"), Is.Null); + Assert.That(run.Client.Fault, Is.Null); + } + + [Test] + public void ASnapshotOfManyDocumentsAppliesEveryOne() + { + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two"), ("c", "three"), ("d", "four"))); + + Assert.That(new[] { "a", "b", "c", "d" }.Select(key => run.ReplicaItem(key)?.Sku), Is.EqualTo(new[] { "one", "two", "three", "four" })); + } + + [Test] + public async Task ManyCommitsInARowAreNotMistakenForAnUnboundedBuffer() + { + using var run = new Replication(source => Stage(source, ("a", "s0"), ("b", "s1"))); + + for (var round = 1; round <= 80; round++) + { + var a = "s" + (round % 2); + var b = "s" + ((round + 1) % 2); + Stage(run.Source, ("a", a), ("b", b)); + } + + Stage(run.Source, ("a", "last-a"), ("b", "last-b")); + await Until(() => run.Client.Fault != null || run.ReplicaItem("a")!.Sku == "last-a"); + Assert.That(run.Client.Fault, Is.Null, "completed commits leave the buffer"); + Assert.That(run.ReplicaItem("b")!.Sku, Is.EqualTo("last-b")); + } + + [Test] + public async Task AMessageDeliveredTwiceInOneCommitFaults() + { + using var run = new Replication(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "3", 3, id: "same")); + run.Peer.Client!.Emit(Put(run, "items", "b", "two", "3", 3, id: "same")); + await Until(() => run.Client.Fault != null); + + Assert.That(run.Faults, Has.Count.EqualTo(1)); + Assert.That(run.ReplicaItem("a"), Is.Null); + } + + [Test] + public async Task ACommitThatChangesItsAnnouncedSizeFaults() + { + using var run = new Replication(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "3", 2)); + run.Peer.Client!.Emit(Put(run, "items", "b", "two", "3", 3)); + await Until(() => run.Client.Fault != null); + + Assert.That(run.ReplicaItem("a"), Is.Null); + } + + [Test] + public async Task ACommitAnnouncingMoreChangesThanAReplicaHoldsFaults() + { + using var run = new Replication(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "3", 1_000_001)); + await Until(() => run.Client.Fault != null); + } + + [Test] + public async Task MoreIncompleteCommitsThanTheBoundFault() + { + using var run = new Replication(); + + for (var commit = 0; commit < 64; commit++) + run.Peer.Client!.Emit(Put(run, "items", "k" + commit, "v" + commit, "c" + commit, 2)); + await Task.Delay(200); + Assert.That(run.Client.Fault, Is.Null, "64 incomplete commits are within the bound"); + Assert.That(Pending(run.Client), Is.EqualTo(64)); + + run.Peer.Client!.Emit(Put(run, "items", "k64", "v64", "c64", 2)); + await Until(() => run.Client.Fault != null); + Assert.That(Pending(run.Client), Is.Zero, "a fault drops what was waiting"); + } + + [Test] + public async Task UnsubscribingDropsTheCommitsStillWaiting() + { + using var run = new Replication(); + var emit = run.Peer.Client!; + + emit.Emit(Put(run, "items", "a", "one", "4", 2)); + await Task.Delay(50); + run.Client.Unsubscribe("items"); + emit.Emit(Put(run, "items", "b", "two", "4", 2)); + await Task.Delay(100); + + Assert.That(run.ReplicaItem("a"), Is.Null); + Assert.That(run.ReplicaItem("b"), Is.Null, "the second half of a dropped commit is not a commit"); + } + + [Test] + public async Task AFaultedClientRefusesANewSubscription() + { + using var run = new Replication(seedReplica: replica => Stage(replica, ("local", "taken"))); + Stage(run.Source, ("a", "taken")); + await Until(() => run.Client.Fault != null); + + var refusal = Assert.Throws(() => run.Client.SubscribeAsync("again")); + Assert.That(refusal!.InnerException, Is.SameAs(run.Client.Fault)); + } + + private static int Pending(CultNetDatabaseSubscriptionClient client) + { + var commits = (System.Collections.IDictionary)typeof(CultNetDatabaseSubscriptionClient) + .GetField("_commits", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!.GetValue(client)!; + lock (commits) + return commits.Count; + } + + [Test] + public async Task TheIncompleteCommitsMayAnnounceOneMillionChangesBetweenThem() + { + using var run = new Replication(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "1", 500_000)); + run.Peer.Client!.Emit(Put(run, "items", "b", "two", "2", 500_000)); + await Task.Delay(200); + Assert.That(run.Client.Fault, Is.Null, "500k and 500k fit"); + + run.Peer.Client!.Emit(Put(run, "items", "c", "three", "3", 2)); + await Until(() => run.Client.Fault != null); + } + + [Test] + public async Task ACommitOfOneMillionChangesIsHeldAndOneMillionAndOneIsNot() + { + using var held = new Replication(); + held.Peer.Client!.Emit(Put(held, "items", "a", "one", "1", 1_000_000)); + await Task.Delay(200); + Assert.That(held.Client.Fault, Is.Null); + Assert.That(Pending(held.Client), Is.EqualTo(1)); + + using var over = new Replication(); + over.Peer.Client!.Emit(Put(over, "items", "a", "one", "1", 1_000_001)); + await Until(() => over.Client.Fault != null); + } + + [Test] + public async Task ACommitThatShrinksItsAnnouncedSizeFaults() + { + using var run = new Replication(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "3", 3)); + run.Peer.Client!.Emit(Put(run, "items", "b", "two", "3", 2)); + await Until(() => run.Client.Fault != null); + } + + [Test] + public async Task UnsubscribingOneSubscriptionKeepsTheCommitsOfAnother() + { + using var run = new Replication(); + run.Client.SubscribeAsync("items2", recordKeys: new[] { "nothing-matches" }).GetAwaiter().GetResult(); + var emit = run.Peer.Client!; + + emit.Emit(Put(run, "items", "a", "one", "4", 2)); + emit.Emit(Put(run, "items2", "b", "two", "4", 2)); + await Task.Delay(100); + Assert.That(Pending(run.Client), Is.EqualTo(2)); + + run.Client.Unsubscribe("items"); + Assert.That(Pending(run.Client), Is.EqualTo(1), "only the unsubscribed subscription's commits leave"); + emit.Emit(Put(run, "items2", "c", "three", "4", 2)); + await Until(() => run.ReplicaItem("b") != null); + Assert.That(run.ReplicaItem("c"), Is.Not.Null); + Assert.That(run.ReplicaItem("a"), Is.Null); + } + + [Test] + public async Task AStragglerAfterUnsubscribeOrFromAnUnknownSubscriptionIsNotBuffered() + { + using var run = new Replication(); + run.Client.Unsubscribe("items"); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "4", 2)); + run.Peer.Client!.Emit(Put(run, "never-subscribed", "b", "two", "4", 2)); + await Task.Delay(150); + + Assert.That(Pending(run.Client), Is.Zero); + Assert.That(run.Client.Fault, Is.Null); + } + + [Test] + public void DisposingTheClientDropsTheCommitsStillWaiting() + { + var run = new Replication(); + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "4", 2)); + SpinWait.SpinUntil(() => Pending(run.Client) == 1, 2000); + Assert.That(Pending(run.Client), Is.EqualTo(1)); + + run.Dispose(); + + Assert.That(Pending(run.Client), Is.Zero); + } + + [Test] + public async Task AFaultedClientBuffersNothingMore() + { + using var run = new Replication(); + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "3", 3, id: "same")); + run.Peer.Client!.Emit(Put(run, "items", "b", "two", "3", 3, id: "same")); + await Until(() => run.Client.Fault != null); + + run.Peer.Client!.Emit(Put(run, "items", "c", "three", "8", 2)); + run.Peer.Client!.Emit(Put(run, "items", "d", "four", "9", 2)); + await Task.Delay(150); + + Assert.That(Pending(run.Client), Is.Zero); + Assert.That(run.Faults, Has.Count.EqualTo(1), "a fault is reported once"); + } + + [Test] + public void AFaultIsReportedOnceHoweverOftenItIsRaised() + { + using var run = new Replication(); + var raise = typeof(CultNetDatabaseSubscriptionClient) + .GetMethod("Break", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!; + + raise.Invoke(run.Client, new object[] { new InvalidOperationException("first") }); + raise.Invoke(run.Client, new object[] { new InvalidOperationException("second") }); + + Assert.That(run.Faults, Has.Count.EqualTo(1)); + Assert.That(run.Client.Fault!.Message, Is.EqualTo("first")); + } + + private static object CommitsOf(CultNetDatabaseSubscriptionClient client) => + typeof(CultNetDatabaseSubscriptionClient) + .GetField("_commits", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!.GetValue(client)!; + + // The check that a subscription is live belongs to the same lock Unsubscribe evicts under. The test holds that lock (it is + // re-entrant), waits until the straggler's handler is contending for it, ends the subscription, and releases: a handler + // that had already passed its check by then would buffer a commit nobody will ever finish. + [Test] + public void AStragglerThatArrivedBeforeUnsubscribeBuffersNothingAfterIt() + { + using var run = new Replication(); + Task tail; + + lock (CommitsOf(run.Client)) + { + var contended = Monitor.LockContentionCount; + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "4", 2)); + Assert.That(SpinWait.SpinUntil(() => Monitor.LockContentionCount > contended, 10_000), "the handler never reached the lock"); + run.Client.Unsubscribe("items"); + tail = (Task)typeof(CultNetDatabaseSubscriptionClient) + .GetField("_queue", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!.GetValue(run.Client)!; + } + tail.GetAwaiter().GetResult(); + + Assert.That(Pending(run.Client), Is.Zero); + Assert.That(run.Client.Fault, Is.Null); + } + + // The 1,000,000 budget is the client's, over every subscription's incomplete commits, not each subscription's own. + [Test] + public async Task TheBudgetIsSharedByAllSubscriptions() + { + using var run = new Replication(); + run.Client.SubscribeAsync("items2", recordKeys: new[] { "nothing-matches" }).GetAwaiter().GetResult(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "1", 600_000)); + run.Peer.Client!.Emit(Put(run, "items2", "b", "two", "2", 400_000)); + await Task.Delay(200); + Assert.That(run.Client.Fault, Is.Null, "600k and 400k fit in one million"); + Assert.That(Pending(run.Client), Is.EqualTo(2)); + + run.Peer.Client!.Emit(Put(run, "items2", "c", "three", "3", 2)); + await Until(() => run.Client.Fault != null); + } + + [Test] + public async Task TwoSubscriptionsAnnouncingMoreThanTheBudgetBetweenThemFault() + { + using var run = new Replication(); + run.Client.SubscribeAsync("items2", recordKeys: new[] { "nothing-matches" }).GetAwaiter().GetResult(); + + run.Peer.Client!.Emit(Put(run, "items", "a", "one", "1", 600_000)); + run.Peer.Client!.Emit(Put(run, "items2", "b", "two", "2", 600_000)); + await Until(() => run.Client.Fault != null); + Assert.That(Pending(run.Client), Is.Zero); + } + + [Test] + public async Task AStragglerFromAnUnknownIdIsDroppedWhileAnotherSubscriptionIsLive() + { + using var run = new Replication(); + run.Client.SubscribeAsync("items2", recordKeys: new[] { "nothing-matches" }).GetAwaiter().GetResult(); + run.Peer.Client!.Emit(Put(run, "items2", "b", "two", "4", 2)); + await Until(() => Pending(run.Client) == 1); + + run.Peer.Client!.Emit(Put(run, "items-ghost", "a", "one", "4", 2)); + run.Peer.Client!.Emit(Put(run, "items-ghost", "c", "three", "5", 3)); + run.Peer.Client!.Emit(Put(run, "item", "e", "five", "6", 2)); + await Task.Delay(200); + + Assert.That(Pending(run.Client), Is.EqualTo(1), "only the live subscription's commit is held"); + Assert.That(run.Client.Fault, Is.Null); + run.Peer.Client!.Emit(Put(run, "items2", "d", "four", "4", 2)); + await Until(() => run.ReplicaItem("b") != null); + Assert.That(run.ReplicaItem("a"), Is.Null); + } + + // Reconcile() can run after the cache admitted a commit and before the subscription's Watch has seen it (the observer + // runs on the writer's thread after the cache gate opens). Its diff then carries changed values, and a swap sent record by + // record faults the replica. The test holds the subscription server's gate to stand in that window. + [Test] + public async Task AReconcileThatRacesACommitSendsItsSwapAsOneCommit() + { + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two"))); + Task write; + + lock (run.Gate) + { + write = Task.Run(() => Stage(run.Source, ("a", "two"), ("b", "one"))); + Assert.That(SpinWait.SpinUntil(() => run.Source.Get(new CultRecordKey("a"))!.Sku == "two", 10_000), + "the source did not take the swap"); + run.Reconcile(); + } + await write; + await Until(() => run.ReplicaItem("a")!.Sku == "two"); + + Assert.That(run.ReplicaItem("b")!.Sku, Is.EqualTo("one")); + Assert.That(run.Client.Fault, Is.Null); + Assert.That(run.Peer.Changes.Select(change => change.CommitCount), Is.All.EqualTo(2)); + } + + [Test] + public async Task ReconcilingAfterAPolicyChangeThatFollowsASwapLeavesTheReplicaIntact() + { + var hidden = new HashSet(); + using var run = new Replication(source => Stage(source, ("a", "one"), ("b", "two")), + authorizeRecord: (_, _, key, _) => !hidden.Contains(key)); + hidden.Add("b"); + + Stage(run.Source, ("a", "two"), ("b", "one")); + await Until(() => run.ReplicaItem("b") == null && run.ReplicaItem("a")!.Sku == "two"); + + hidden.Remove("b"); + hidden.Add("a"); + run.Reconcile(); + await Until(() => run.ReplicaItem("b") != null && run.ReplicaItem("a") == null); + + Assert.That(run.ReplicaItem("b")!.Sku, Is.EqualTo("one")); + Assert.That(run.Client.Fault, Is.Null); + } + + // Source, wire and client all real: a CultNetDatabaseServer behind a Server on a loopback socket, a LiteNetLib client + // standing in for the transport of a CultNetDatabaseSubscriptionClient. No handler is called directly. + private sealed class SocketClient : ICultNetSchemaClient + { + private readonly NetManager _manager; + private readonly NetPeer _peer; + private readonly List> _snapshots = new(); + private readonly List> _changes = new(); + + public SocketClient(int port, string key) + { + var listener = new EventBasedNetListener(); + var connected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var decoder = new LiteNetLibTransportConnection(); + listener.PeerConnectedEvent += _ => connected.TrySetResult(true); + listener.NetworkReceiveEvent += (_, reader, _, _) => + { + switch (LiteNetLibTransportConnection.DecodeSchema(decoder.Receive(reader.GetRemainingBytes()))) + { + case CultNetSnapshotResponseRawMessage snapshot: foreach (var handler in _snapshots.ToArray()) handler(snapshot); break; + case CultNetDatabaseChangeRawMessage change: foreach (var handler in _changes.ToArray()) handler(change); break; + } + }; + _manager = new NetManager(listener) { UnsyncedEvents = true }; + _manager.Start(0); + _peer = _manager.Connect("127.0.0.1", port, key); + if (!connected.Task.Wait(TimeSpan.FromSeconds(10))) Assert.Fail("the client did not connect"); + } + + public bool Connected => _peer.ConnectionState == ConnectionState.Connected; + public void Connect(string host, int port) { } + public void SendCultNet(T message) where T : ICultNetSchemaMessage => + _peer.Send(CultNetSchemaMessageSerialization.Serialize(message), DeliveryMethod.ReliableOrdered); + public void OnCultNet(Action callback) where T : ICultNetSchemaMessage + { + if (callback is Action snapshot) _snapshots.Add(snapshot); + else if (callback is Action change) _changes.Add(change); + } + public void Dispose() => _manager.Stop(); + } + + [Test] + public async Task ASwapThroughTheRealDatabaseServerReachesTheClientIntact() + { + var source = new CultCache(); + Stage(source, ("a", "one"), ("b", "two")); + using var database = new CultNetDatabase(source); + using var server = new Server(source, ServerSecurityOptions.Development()) { Port = 0 }; + using var databaseServer = new CultNetDatabaseServer(server, database); + server.Start(); + var replica = new CultCache(); + var documents = new CultNetDocumentRegistry(replica.Registry) + .Register(CultNetDocumentBinding.ForDocument(replica.Registry)); + using var client = new CultNetDatabaseSubscriptionClient( + new SocketClient(server.Port, ServerSecurityOptions.Development().ConnectionKey), replica, documents); + var faults = new List(); + client.Faulted += faults.Add; + await client.SubscribeAsync("items", schemaIds: new[] { source.Registry.GetRequired().SchemaId }); + Assert.That(replica.Get(new CultRecordKey("a"))!.Sku, Is.EqualTo("one")); + + Stage(source, ("a", "two"), ("b", "one")); + await Until(() => replica.Get(new CultRecordKey("a"))!.Sku == "two"); + + Assert.That(replica.Get(new CultRecordKey("b"))!.Sku, Is.EqualTo("one")); + Assert.That(replica.GetByIndex("sku", "one"), Is.SameAs(replica.Get(new CultRecordKey("b")))); + Assert.That(client.Fault, Is.Null); + Assert.That(faults, Is.Empty); + } + + [TestCase(5000)] + [TestCase(100_000)] + public async Task ACommitOfManyChangesThroughTheRealSubscriptionServerAppliesIntact(int size) + { + using var run = new Replication(); + var keys = Enumerable.Range(0, size).Select(i => ("k" + i, "s" + i)).ToArray(); + + Stage(run.Source, keys); + await Until(() => run.ReplicaItem("k" + (size - 1)) != null || run.Client.Fault != null, seconds: 180); + + Assert.That(run.Client.Fault, Is.Null); + Assert.That(run.Replica.GetAll().Count(), Is.EqualTo(size)); + Assert.That(run.Peer.Changes.Select(change => change.CommitCount).Distinct().Single(), Is.EqualTo(size)); + Assert.That(run.ReplicaItem("k" + (size / 2))!.Sku, Is.EqualTo("s" + (size / 2))); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs index 4172c83b..ec9dde5b 100644 --- a/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSchemaContractTests.cs @@ -1,5 +1,7 @@ #nullable enable using System; +using System.Buffers; +using System.Collections.Generic; using System.IO; using System.Text.Json; using System.Threading.Tasks; @@ -246,6 +248,39 @@ namespace GameCult.Networking.Tests Assert.That(Convert.ToHexString(CultNetSchemaMessageSerialization.Serialize(message)), Is.EqualTo("85AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7630A56572726F72D93E6D6573736167655F696E76616C69643A2052617720707574206D65737361676573206D75737420636F6E7461696E206120646F63756D656E74206D61702EAB726F7574696E6748696E74C0A4636F6465AF6D6573736167655F696E76616C6964A764657461696C7384A56669656C64A8646F63756D656E74A576616C7565C0A461734F66C0A763757272656E74C0")); } + // The C# writer sets every key (nil where unset), so a removed change carries document: nil, and an added change + // carries recordKey/schemaId: nil. The schema describes those bytes; it still refuses an added change with no document. + private static List CommitVectorChanges() + { + var path = Path.Combine(RepoRoot(), "tests", "vectors", "database-change-commit", "two-change-commit.msgpack"); + var reader = new MessagePackReader(File.ReadAllBytes(path)); + var changes = new List(); + while (!reader.End) + changes.Add(JsonDocument.Parse(MessagePackSerializer.ConvertToJson(reader.ReadRaw().ToArray())).RootElement); + return changes; + } + + [Test] + public void DatabaseChangeRaw_TheCommitVectorsRemovedChangesWithANilDocument_MatchTheSchema() + { + var validator = new MiniJsonSchemaValidator(SchemaDir()); + var changes = CommitVectorChanges(); + Assert.That(changes, Has.Count.EqualTo(2)); + foreach (var change in changes) + { + Assert.That(change.GetProperty("document").ValueKind, Is.EqualTo(JsonValueKind.Null), "the C# writer wrote the key, nil"); + validator.AssertValid(change, "cultnet.database-change-raw.schema.json"); + } + } + + [Test] + public void DatabaseChangeRaw_AnAddedChangeWithANilDocument_IsRefused() + { + var validator = new MiniJsonSchemaValidator(SchemaDir()); + var removed = CommitVectorChanges()[0].GetRawText().Replace("\"removed\"", "\"added\""); + Assert.Throws(() => validator.AssertValid(JsonDocument.Parse(removed).RootElement, "cultnet.database-change-raw.schema.json")); + } + private static void AssertErrorMatchesSchema(CultNetErrorMessage message) { var validator = new MiniJsonSchemaValidator(SchemaDir()); diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionEvaluatorTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionEvaluatorTests.cs index b2dfcf3c..b700ec19 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionEvaluatorTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionEvaluatorTests.cs @@ -486,6 +486,39 @@ namespace GameCult.Networking.Tests Assert.That(evaluation.Rows.Select(r => r.Key.Value), Is.EqualTo(new[] { "a" })); } + // [CultIndex] means unique and is also a selection address: a unique member filters by equality and by number like a role. + [Test] + public void AUniqueIndexMemberIsAlsoSelectable() + { + var registry = CultDocumentRegistry.ForTypes(new[] { typeof(SelUnique) }); + var rows = new[] + { + Row(registry, new SelUnique { Sku = "s1", Rank = 1 }, "a", 1), + Row(registry, new SelUnique { Sku = "s2", Rank = 7 }, "b", 2) + }; + + var byValue = CultNetSelectionEvaluator.Select(registry, rows, + new CultNetSelection { Fields = new[] { new CultNetFieldPredicate { Index = "sku", Op = "any_of", Values = new[] { "s2" } } } }, asOf: 2); + var byNumber = CultNetSelectionEvaluator.Select(registry, rows, + new CultNetSelection { Fields = new[] { new CultNetFieldPredicate { Index = "rank", Op = "gt", Number = "3" } } }, asOf: 2); + + Assert.That(byValue.Rows.Select(r => r.Key.Value), Is.EqualTo(new[] { "b" })); + Assert.That(byNumber.Rows.Select(r => r.Key.Value), Is.EqualTo(new[] { "b" })); + } + + [CultDocument("cultnet.selection-tests.unique", "cultnet.selection-tests.unique.v1")] + [MessagePackObject(AllowPrivate = true)] + public sealed class SelUnique + { + [Key(0)] + [CultIndex("sku")] + public string Sku = string.Empty; + + [Key(1)] + [CultIndex("rank")] + public int Rank; + } + public abstract class SelFixtureMiddle { [Key(0)] @@ -493,11 +526,11 @@ namespace GameCult.Networking.Tests public string Name = string.Empty; [Key(1)] - [CultIndex("mass")] + [CultRole("mass")] public float Mass; [Key(2)] - [CultIndex("kind")] + [CultRole("kind")] public string Kind = string.Empty; } diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs index 9ecc4556..030ae100 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs @@ -581,7 +581,7 @@ namespace GameCult.Networking.Tests return instance; } - // A plain data member "Owner" with no [CultIndex] (its role falls back to its bare name) + // A plain data member "Owner" with no [CultRole] (its role falls back to its bare name) // and a reference member whose alias is explicitly "Owner" - the exact shape Soul probed. var ownerMember = MakeMember(typeof(GuardProbeShape).GetField(nameof(GuardProbeShape.Owner))!, typeof(string), 0, indexAlias: null, isReference: false, targetType: null); diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionPageTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionPageTests.cs index 0f1f20a9..845f9468 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionPageTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionPageTests.cs @@ -130,7 +130,7 @@ namespace GameCult.Networking.Tests public string Name = string.Empty; [Key(1)] - [CultIndex("kind")] + [CultRole("kind")] public string Kind = string.Empty; } diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionSurvivorTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionSurvivorTests.cs index a03b62d2..e625c134 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionSurvivorTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionSurvivorTests.cs @@ -62,7 +62,7 @@ namespace GameCult.Networking.Tests public sealed class SurvivorNumericLeaf { [Key(0)] [CultName] public string Name = string.Empty; - [Key(1)] [CultIndex("level")] public int Level; + [Key(1)] [CultRole("level")] public int Level; } [CultDocument("cultnet.selection-tests.survivor_string_leaf", "cultnet.selection-tests.survivor_string_leaf.v1")] @@ -70,7 +70,7 @@ namespace GameCult.Networking.Tests public sealed class SurvivorStringLeaf { [Key(0)] [CultName] public string Name = string.Empty; - [Key(1)] [CultIndex("level")] public string Level = string.Empty; + [Key(1)] [CultRole("level")] public string Level = string.Empty; } // --------------------------------------------------------------------------------------------- @@ -173,7 +173,7 @@ namespace GameCult.Networking.Tests // The declared index alias ("linkRole") differs from the member name ("Link") - the exact // shape section 2's role-naming rule exists for, and no prior fixture had one. [Key(1)] - [CultIndex("linkRole")] + [CultRole("linkRole")] [CultReference(typeof(SurvivorAliasTarget))] public CultRecordRef Link; } @@ -408,7 +408,7 @@ namespace GameCult.Networking.Tests [Key(0)] [CultName] public string Name = string.Empty; [Key(1)] - [CultIndex("hopRole")] + [CultRole("hopRole")] [CultReference(typeof(SurvivorAliasTarget))] public CultRecordRef Link; } @@ -431,12 +431,12 @@ namespace GameCult.Networking.Tests [Key(0)] [CultName] public string Name = string.Empty; [Key(1)] - [CultIndex("zLastRole")] + [CultRole("zLastRole")] [CultReference(typeof(SurvivorAliasTarget))] public CultRecordRef RefZ; [Key(2)] - [CultIndex("aFirstRole")] + [CultRole("aFirstRole")] [CultReference(typeof(SurvivorAliasTarget))] public CultRecordRef RefA; } diff --git a/tests/GameCult.Networking.Tests/GovernanceVectorTests.cs b/tests/GameCult.Networking.Tests/GovernanceVectorTests.cs index 11230860..57e1cf84 100644 --- a/tests/GameCult.Networking.Tests/GovernanceVectorTests.cs +++ b/tests/GameCult.Networking.Tests/GovernanceVectorTests.cs @@ -303,6 +303,7 @@ namespace GameCult.Networking.Tests Assert.That(Errors(ForkSchema, "refusal", v!["message"]), Is.Empty, S(v["name"])); Assert.That(HexOf(v["message"]), Is.EqualTo(S(v["messagePackHex"])), S(v["name"])); } + Assert.That(Fork["doorRefusals"]!.AsArray().Single(r => S(r!["name"]) == "fork_too_large")!["httpStatus"]!.GetValue(), Is.EqualTo(422)); } [Test] @@ -532,6 +533,20 @@ namespace GameCult.Networking.Tests if ((record["counted_answer_id"] is null ? null : S(record["counted_answer_id"])) != counted) yield return "counted_answer_id is not derived"; } + // Order is a unique rank per Mind: the refusal names both fork ids, the first holder and the second. + private static List OrderErrors(IEnumerable cards) + { + var holder = new Dictionary(); + var errors = new List(); + foreach (var c in cards) + { + var order = c!["order"]!.GetValue(); + if (holder.TryGetValue(order, out var first)) errors.Add($"order {order} is shared by {first} and {S(c["fork_id"])}"); + else holder[order] = S(c["fork_id"]); + } + return errors; + } + // The reference list judge: newest first by card order, status filter, cursor c = cards below order N, a page of limit, counts over the whole set. private static JsonNode JudgeList(JsonArray entries, long ordinal, JsonNode request) { @@ -636,7 +651,9 @@ namespace GameCult.Networking.Tests } foreach (var v in t["listInvalid"]!.AsArray()) { - Assert.That(Errors(ForkSchema, "list", v!["message"]), Is.Not.Empty, $"{S(v["name"])}: {S(v["why"])}"); + var shared = OrderErrors(v!["message"]!["forks"]!.AsArray().Select(f => f!["card"])); + Assert.That(Errors(ForkSchema, "list", v["message"]).Count > 0 || shared.Count > 0, Is.True, $"{S(v["name"])}: {S(v["why"])}"); + if (v["refusal"] is not null) Assert.That(shared, Is.EqualTo(new[] { S(v["refusal"]) }), S(v["name"])); Assert.That(HexOf(v["message"]), Is.EqualTo(S(v["messagePackHex"])), S(v["name"])); } } @@ -649,6 +666,7 @@ namespace GameCult.Networking.Tests var name = S(sc!["name"]); var entries = sc["entries"]!.AsArray(); foreach (var e in entries) Assert.That(EntryRuleErrors(e!), Is.Empty, name); + Assert.That(OrderErrors(entries.Select(e => e!["card"])), Is.Empty, name); JsonNode? previous = null; foreach (var step in sc["steps"]!.AsArray()) { @@ -664,6 +682,122 @@ namespace GameCult.Networking.Tests } } + [Test] + public void ForkTransport_OrderZero_IsLegal_NoCursorNamesIt_AndThePageEndingAtOrderOneHandsOnToIt() + { + var sc = Fork["transport"]!["listScenarios"]!.AsArray().Single(x => S(x!["name"]) == "order_zero_is_legal_but_no_cursor_names_it")!; + var steps = sc["steps"]!.AsArray(); + Assert.That(S(steps[0]!["response"]!["next_cursor"]), Is.EqualTo("c1")); + Assert.That(steps[1]!["response"]!["next_cursor"], Is.Null); + Assert.That(steps[1]!["response"]!["forks"]!.AsArray().Select(f => f!["card"]!["order"]!.GetValue()), Is.EqualTo(new[] { 0L })); + Assert.That(Errors(ForkSchema, "cursor", JsonValue.Create("c0")), Is.Not.Empty); + } + + [Test] + public void ForkTransport_AShortPage_AndForkTooLarge422_ContinuedBelowItsOrder() + { + var sc = Fork["transport"]!["listShortPages"]!.AsArray(); + Assert.That(sc.Select(x => S(x!["name"])), Is.EqualTo(new[] { "a_short_page_resumes_below_its_last_entry", "a_first_entry_past_the_budget_is_fork_too_large_and_the_client_continues_below_its_order", "fork_too_large_on_order_zero_leaves_nothing_below" })); + var schema = JsonNode.Parse(File.ReadAllText(Path.Combine(RepoRoot(), "contracts", "cultnet", ForkSchema)))!; + Assert.That(schema["$defs"]!["refusal"]!["properties"]!["code"]!["enum"]!.AsArray().Select(S), Does.Contain("fork_too_large")); + var covered = new List(); + foreach (var s in sc) + { + var entries = s!["entries"]!.AsArray(); var steps = s["steps"]!.AsArray(); string? last = null; var follow = false; + for (var i = 0; i < steps.Count; i++) + { + var step = steps[i]!; var name = S(s["name"]); + if (follow) Assert.That(step["request"]!["cursor"] is null ? null : S(step["request"]!["cursor"]), Is.EqualTo(last), name); + var full = JudgeList(entries, s["ordinal"]!.GetValue(), step["request"]!); + var fullForks = full["forks"]!.AsArray(); + if (step["response"] is not null) + { + var r = step["response"]!; var forks = r["forks"]!.AsArray(); var n = forks.Count; + Assert.That(Errors(ForkSchema, "list", r), Is.Empty, name); + Assert.That(HexOf(r), Is.EqualTo(S(step["responseHex"])), name); + Assert.That(n, Is.GreaterThanOrEqualTo(1), name); + for (var k = 0; k < n; k++) Assert.That(JsonNode.DeepEquals(forks[k], fullForks[k]), Is.True, $"{name}: a prefix of the unbudgeted page"); + var o = forks[n - 1]!["card"]!["order"]!.GetValue(); + if (full["next_cursor"] is not null || n < fullForks.Count) Assert.That(S(r["next_cursor"]), Is.EqualTo($"c{o}"), name); else Assert.That(r["next_cursor"], Is.Null, name); + Assert.That(S(r["stamp"]), Is.EqualTo(S(full["stamp"]))); Assert.That(r["open_count"]!.GetValue(), Is.EqualTo(full["open_count"]!.GetValue())); Assert.That(r["recorded_count"]!.GetValue(), Is.EqualTo(full["recorded_count"]!.GetValue())); + covered.AddRange(forks.Select(f => f!["card"]!["order"]!.GetValue())); + last = r["next_cursor"] is null ? null : S(r["next_cursor"]); follow = true; + } + else + { + var r = step["refusal"]!; var first = fullForks[0]!["card"]!; + Assert.That(Errors(ForkSchema, "refusal", r), Is.Empty, name); + Assert.That(HexOf(r), Is.EqualTo(S(step["refusalHex"])), name); + Assert.That(S(r["code"]), Is.EqualTo("fork_too_large")); Assert.That(step["httpStatus"]!.GetValue(), Is.EqualTo(422)); + Assert.That(S(step["fork_id"]), Is.EqualTo(S(first["fork_id"]))); Assert.That(step["order"]!.GetValue(), Is.EqualTo(first["order"]!.GetValue())); + Assert.That(S(r["error"]), Does.Contain(S(first["fork_id"])), name); + Assert.That(System.Text.RegularExpressions.Regex.IsMatch(S(r["error"]).Replace(S(first["fork_id"]), ""), $@"\b{first["order"]!.GetValue()}\b"), Is.True, $"{name}: the text names the order"); + Assert.That(step["request"]!["limit"]!.GetValue(), Is.GreaterThanOrEqualTo(1)); + var order = first["order"]!.GetValue(); + if (order >= 1) { Assert.That(i + 1, Is.LessThan(steps.Count), name); last = $"c{order}"; follow = true; Assert.That(Errors(ForkSchema, "cursor", JsonValue.Create(last)), Is.Empty); } + else { Assert.That(i, Is.EqualTo(steps.Count - 1), $"{name}: order 0 is the end"); follow = false; } + } + } + } + Assert.That(covered, Is.EqualTo(new long[] { 3, 2, 1, 0, 2, 1, 0, 1 })); + Assert.That(sc[0]!["steps"]!.AsArray().Select(x => x!["response"]!["forks"]!.AsArray().Count), Is.EqualTo(new[] { 2, 2 })); + } + + [Test] + public void ForkTransport_AReadWhoseEntryIsPastTheCap_IsForkTooLarge422_NamedByForkIdAlone() + { + var doorStatus = Fork["doorRefusals"]!.AsArray().Single(r => S(r!["name"]) == "fork_too_large")!["httpStatus"]!.GetValue(); + var cases = Fork["transport"]!["readTooLarge"]!.AsArray(); + Assert.That(cases.Count, Is.EqualTo(1)); + foreach (var c in cases) + { + var held = c!["entries"]!.AsArray().Single(e => S(e!["card"]!["fork_id"]) == S(c["request"]!["fork_id"]))!; + Assert.That(Errors(ForkSchema, "readRequest", c["request"]), Is.Empty, S(c["name"])); + Assert.That(Errors(ForkSchema, "refusal", c["refusal"]), Is.Empty, S(c["name"])); + Assert.That(S(c["refusal"]!["code"]), Is.EqualTo("fork_too_large")); + Assert.That(S(c["refusal"]!["error"]), Does.Contain(S(held["card"]!["fork_id"])), "the text names the fork id"); + Assert.That(HexOf(c["refusal"]), Is.EqualTo(S(c["refusalHex"])), S(c["name"])); + Assert.That(c["httpStatus"]!.GetValue(), Is.EqualTo(422)); Assert.That(c["httpStatus"]!.GetValue(), Is.EqualTo(doorStatus)); + } + } + + [Test] + public void ForkTransport_AStaleAnswerWhoseCardRereadIsPastTheCap_IsTheReadFormForkTooLarge422_OverAStoredStaleRefusal() + { + var cases = Fork["transport"]!["answerTooLarge"]!.AsArray(); + Assert.That(cases.Count, Is.EqualTo(1)); + var read = Fork["transport"]!["readTooLarge"]![0]!; + foreach (var c in cases) + { + var held = c!["entries"]!.AsArray().Single(e => S(e!["card"]!["fork_id"]) == S(c["request"]!["fork_id"]))!; + Assert.That(Errors(ForkSchema, "answerRequest", c["request"]), Is.Empty, S(c["name"])); + Assert.That(S(c["request"]!["options_hash"]), Is.Not.EqualTo(S(held["card"]!["options_hash"])), "the card moved: the answer is stale"); + Assert.That(Errors(ForkSchema, "refusal", c["refusal"]), Is.Empty, S(c["name"])); + Assert.That(S(c["refusal"]!["code"]), Is.EqualTo("fork_too_large")); + Assert.That(S(c["refusal"]!["error"]), Does.Contain(S(held["card"]!["fork_id"])), "the text names the fork id"); + Assert.That(HexOf(c["refusal"]), Is.EqualTo(S(c["refusalHex"])), S(c["name"])); + Assert.That(S(c["refusal"]!["code"]), Is.EqualTo(S(read["refusal"]!["code"])), "the same refusal as the read form"); + Assert.That(c["httpStatus"]!.GetValue(), Is.EqualTo(422)); + Assert.That(c["httpStatus"]!.GetValue(), Is.EqualTo(Fork["doorRefusals"]!.AsArray().Single(r => S(r!["name"]) == "fork_too_large")!["httpStatus"]!.GetValue())); + Assert.That(S(c["outcome"]), Is.EqualTo("Refused")); Assert.That(S(c["code"]), Is.EqualTo("stale_fork")); Assert.That(c["counted"]!.GetValue(), Is.False); + } + } + + [Test] + public void ForkTransport_ADoorRequestIdConflict_IsAFaultOnReadAndList_AndARefusalOnlyOnTheAnswer() + { + var cases = Fork["transport"]!["doorFaults"]!.AsArray(); + Assert.That(cases.Select(c => S(c!["route"])), Is.EqualTo(new[] { "read", "list", "answer" })); + foreach (var c in cases) + { + var source = S(c!["route"]) == "answer" ? Fork["doorRefusals"]!.AsArray().Single(r => S(r!["name"]) == S(c["doorCode"]))! : Fork["unavailable"]![0]!; + Assert.That(S(c["responseHex"]), Is.EqualTo(S(source["messagePackHex"])), S(c["name"])); + Assert.That(HexOf(c["response"]), Is.EqualTo(S(c["responseHex"])), S(c["name"])); + Assert.That(c["httpStatus"]!.GetValue(), Is.EqualTo(source["httpStatus"]!.GetValue()), S(c["name"])); + Assert.That(Errors(ForkSchema, S(c["route"]) == "answer" ? "refusal" : "unavailable", c["response"]), Is.Empty, S(c["name"])); + } + } + [Test] public void ForkTransport_Auth_BearerThenAudienceThenScope_AVotesTokenTouchesNothing() { diff --git a/tests/GameCult.Networking.Tests/SelectionParityVectorTests.cs b/tests/GameCult.Networking.Tests/SelectionParityVectorTests.cs index e9a070fc..3a076d16 100644 --- a/tests/GameCult.Networking.Tests/SelectionParityVectorTests.cs +++ b/tests/GameCult.Networking.Tests/SelectionParityVectorTests.cs @@ -586,11 +586,11 @@ namespace GameCult.Networking.Tests public string Name = string.Empty; [Key(1)] - [CultIndex("kind")] + [CultRole("kind")] public string Kind = string.Empty; [Key(2)] - [CultIndex("mass")] + [CultRole("mass")] public float Mass; } @@ -631,7 +631,7 @@ namespace GameCult.Networking.Tests public string Name = string.Empty; [Key(1)] - [CultIndex("narrow_ref")] + [CultRole("narrow_ref")] [CultReference(typeof(ParityLeafA))] public CultRecordRef NarrowRef; } diff --git a/tests/vectors/database-change-commit/README.md b/tests/vectors/database-change-commit/README.md new file mode 100644 index 00000000..39a78fa1 --- /dev/null +++ b/tests/vectors/database-change-commit/README.md @@ -0,0 +1,10 @@ +# database-change-commit vectors + +Written only by the C# reference, `ATwoChangeCommitIsWrittenAsTheVectorOtherRuntimesDecode` in +`tests/GameCult.Networking.Tests/CultNetReplicaCommitTests.cs`, with `CULTCACHE_WRITE_VECTORS=1`. Every other run +compares the bytes. + +- `two-change-commit.msgpack`: two `cultnet.database_change_raw.v0` messages (`removed`, `note:a` and `note:b`) + concatenated, each as the C# `[MessagePackObject]` writes it (every key present, nil where unset), both carrying + `commitId` `9f1c0f0e4b7a4c1f9d2e` and `commitCount` 2. Read it as a msgpack stream. Python walks it in + `packages/cultnet-py/tests/test_cultnet.py`. diff --git a/tests/vectors/database-change-commit/two-change-commit.msgpack b/tests/vectors/database-change-commit/two-change-commit.msgpack new file mode 100644 index 00000000..055dc4ae --- /dev/null +++ b/tests/vectors/database-change-commit/two-change-commit.msgpack @@ -0,0 +1 @@ +‰­schemaVersion¾cultnet.database_change_raw.v0©messageId¨change-0®subscriptionId¥sub-1ªchangeKind§removed¨documentÀ©recordKey¦note:a¨schemaId«schema-note¨commitId´9f1c0f0e4b7a4c1f9d2e«commitCount‰­schemaVersion¾cultnet.database_change_raw.v0©messageId¨change-1®subscriptionId¥sub-1ªchangeKind§removed¨documentÀ©recordKey¦note:b¨schemaId«schema-note¨commitId´9f1c0f0e4b7a4c1f9d2e«commitCount \ No newline at end of file diff --git a/tests/vectors/unique-index/README.md b/tests/vectors/unique-index/README.md new file mode 100644 index 00000000..55b393f3 --- /dev/null +++ b/tests/vectors/unique-index/README.md @@ -0,0 +1,83 @@ +# unique-index vectors + +Written only by the C# reference, `tests/GameCult.Caching.Tests/UniqueIndexVectorTests.cs`, with +`CULTCACHE_WRITE_VECTORS=1`. Every other run reads these files and compares them byte for byte; a changed vector is a +reviewed commit. Other runtimes are held to the same canonical text and the same verdicts (`variants:ruling:unique-indexes`). + +## Files + +- `values.msgpack`: an array of `[label, kind, value, text-or-nil]`, kind one of `string`, `int64`, `uint64`, `double`, + `bool` (decode integers 64-bit wide). `text` is the canonical index text of a member holding `value`: a record holding + it is found by `GetByIndex(alias, text)`. A nil text is a blank value: no value, never indexed, never judged, found by + no text, and two records holding it commit. Strings are as is (never trimmed), blank being empty or only C# + `White_Space` characters (U+0009-000D, U+0020, U+0085, U+00A0, U+1680, U+2000-200A, U+2028, U+2029, U+202F, U+205F, + U+3000); U+FEFF and U+001F are values. Booleans are `True` and `False`. Integers are their decimal digits. A + non-integral finite double is its exact decimal expansion (written by C# `RenderExactDouble`, never hand-typed), + `-0.0` is `0`, NaN and infinities are nil. +- `duplicate-store.cc`: a single-file store, schema `tests.unique_item` v1, slots `[Sku: string, Category: string]`, + stored-at pinned to `2026-01-01T00:00:00Z`. Records `item:a` and `item:b` both hold Sku `S-1`; `item:c` holds Sku + `" "` and `item:d` holds Sku `U+0085`. Written through a role Sku. +- `blank-store.cc`: the same, only `item:c` and `item:d`. +- `pair-store.cc`, `surrogate-store.cc`, `types-store.cc`, `repull-before.cc`, `repull-after.cc`, `alias-store.cc`, + `key-case-store.cc`, `key-punctuation-store.cc`, `surrogate-groups-store.cc`, `holder-case-store.cc`, `cross-schema-store.cc`, `distinct-values-store.cc`, `normal-keys-store.cc`, `normal-three-store.cc`, `wide-store.cc`: single-file stores written like + `duplicate-store.cc`, through roles, for the rows of `refusals.msgpack` that open or pull them. Schemas + `tests.unique_item`, `tests.uq_pair` (Sku and Alt both unique), `tests.uq_alias` (Sku and `alt` both unique), + `tests.uq_wide` (Sku and Alt both unique, aliases U+FF21 and U+10400), `tests.uq_alpha` and `tests.uq_beta`, each slots `[0: Sku string, 1: second string]`, schema version the name plus + `.v1`. The schema ids C# writes in the store catalogs and on every record: + - `tests.unique_item`: `sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882` + - `tests.uq_pair`: `sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60` + - `tests.uq_alias`: `sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f` + - `tests.uq_wide`: `sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f` + - `tests.uq_alpha`: `sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389` + - `tests.uq_beta`: `sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586` +- `outcomes.msgpack`: rows `[file, unique alias or nil, open|refused, k1 or nil, k2 or nil]`. Opening `file` with that + alias a unique index (nil: a role, no unique alias) yields the verdict. A refusal reads, byte for byte, + `Records and (tests.unique_item) both hold unique index ''; an index value names one record.` + with `k1 < k2` ordinal, never contains the value, and leaves the file byte-identical. + +## refusals.msgpack + +RULE: a unique-index refusal is a function of the state the admission would leave, never of staging, file or pull +order. Over every (type, unique alias, non-blank value) that state holds at two or more distinct keys, name the one +least by (alias, its least key, its second-least key, schema name), each compared by UTF-16 code unit (C# +`StringComparer.Ordinal`): `Records and () both hold unique index ''; an index value names one +record.` with `k1 < k2` its two least keys. One rule for a single write, a batch, a load, a re-pull and an index +registration. + +Rows `[label, ops, refusal]`. Each op is `["open", file-or-nil]` (copy that vector file to a fresh path, or nil for a +fresh path with no file, and open a cache on it), `["pull", file]` (overwrite the open cache's file with that vector +file, then pull every backing store again) or `["commit", [[key, schema, sku, second], ...]]` (one batch, staged in row +order; schema is the schema name). `refusal` is nil (every op succeeds) or `[k1, k2, schema, alias]`: every op but the +last succeeds and the last refuses with exactly the RULE text built from those fields, leaving the file +byte-identical. Schemas are those of the store files above; `second` is `Category` (plain) except in `tests.uq_pair` +and `tests.uq_alias`, where it is `Alt` (unique, alias `Alt` and `alt` respectively). + +The 13 rows of the spec are a floor; rows 14-37 extend it with orderings that a code-point, case-insensitive or +culture compare would flip. Rows, in file order: 1 batch-later-value-staged-first, 2 batch-surrogate-keys, 3 +batch-interleaved-types, 4 batch-two-aliases, 5 held-then-landing, 6 held-in-the-least-group, 7 swap-off-a-held-value, +8 load-two-aliases, 9 load-surrogate-keys, 10 load-interleaved-types, 11 repull-held-pair, 12 load-repull-after, 13 +commit-onto-a-loaded-holder, 14 batch-key-case (`B` before `a`), 15 batch-key-punctuation (`Z` before `_` before `a`), +16 batch-key-case-in-a-group, 17 batch-surrogate-groups (U+1F600 before U+E000, the reverse of code-point order), 18 +batch-alias-case (alias `Sku` before `alt`), 19 load-alias-case, 20 load-key-case, 21 load-key-punctuation, 22 +load-surrogate-groups, 23 batch-holders-by-case (`a` and `A` holding one value: `A` before `a`), 24 load-holders-by-case, 25 batch-replaced-by-case (held `a`, then a commit of `A` on its value: `A` does not replace `a`), +26 batch-value-in-two-schemas (one value in `tests.uq_alpha` and `tests.uq_beta` at two keys: two groups, success), 27 +load-value-in-two-schemas, 28 batch-distinct-values (`S`, `s`, `S `, ` S`, U+00E9, `e`+U+0301, U+FF33 are seven values: success), 29 +load-distinct-values, 30 batch-keys-by-normal-form (keys U+00E9 and `e`+U+0301 holding one value are two holders, `e`+U+0301 first), +31 load-keys-by-normal-form, 32 put-onto-a-held-pair (held `m` and landing `a` share Sku and Alt in `tests.uq_pair`, one record per +commit: `Alt` is named before `Sku`), 33 put-onto-a-held-alias-pair (the same in `tests.uq_alias`: `Sku` before `alt`), 34 batch-three-holders-normal-form +(keys `e`+U+0302, `f` and U+00E9 holding one value: the two least by code unit, `e`+U+0302 and `f`, are named; an NFC, NFD or culture sort of the keys names another pair), 35 +load-three-holders-normal-form, 36 batch-alias-code-unit (two records each holding the same value under both aliases of `tests.uq_wide`, U+FF21 and U+10400: by UTF-16 code +unit U+10400 (D801 DC00) precedes U+FF21, by code point it follows it, so the refusal names alias U+10400), 37 load-alias-code-unit. +Rows 19-22, 24, 27, 29, 31, 35 and 37 open a store holding the records of rows 18, 14, 15, 17, 23, 26, 28, 30, 34 and 36, so a runtime with no batch +commit still walks alias and cross-group key order. + +A runtime performs every row its API can perform and asserts the count it walks. These are the counts a runtime must +assert, not a statement of which runtimes walk them today: + +- C# and Rust: all 37. +- Python: 35, every row but 3 and 26 (`put_envelopes` takes one type, so no batch of two types). +- TypeScript: 20, the rows with no multi-record commit: 5, 8, 9, 10, 11, 12, 13, 19, 20, 21, 22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 + (no batch commit, so no row 1-4, 6, 7, 14-18, 23, 26, 28, 30, 34 or 36; `variants:idea:unique-index-runtimes.ts-batch-commit`). + +Not pinned: the schema-name tiebreak. Record keys are unique across a whole cache, so two types never hold the same key and +two groups can never tie on alias and key pair; the final `schema` comparison cannot decide a refusal. diff --git a/tests/vectors/unique-index/alias-store.cc b/tests/vectors/unique-index/alias-store.cc new file mode 100644 index 00000000..3ea51ef5 Binary files /dev/null and b/tests/vectors/unique-index/alias-store.cc differ diff --git a/tests/vectors/unique-index/blank-store.cc b/tests/vectors/unique-index/blank-store.cc new file mode 100644 index 00000000..2b69facc Binary files /dev/null and b/tests/vectors/unique-index/blank-store.cc differ diff --git a/tests/vectors/unique-index/cross-schema-store.cc b/tests/vectors/unique-index/cross-schema-store.cc new file mode 100644 index 00000000..1dd2fb66 Binary files /dev/null and b/tests/vectors/unique-index/cross-schema-store.cc differ diff --git a/tests/vectors/unique-index/distinct-values-store.cc b/tests/vectors/unique-index/distinct-values-store.cc new file mode 100644 index 00000000..6eabdd96 Binary files /dev/null and b/tests/vectors/unique-index/distinct-values-store.cc differ diff --git a/tests/vectors/unique-index/duplicate-store.cc b/tests/vectors/unique-index/duplicate-store.cc new file mode 100644 index 00000000..4caec451 Binary files /dev/null and b/tests/vectors/unique-index/duplicate-store.cc differ diff --git a/tests/vectors/unique-index/holder-case-store.cc b/tests/vectors/unique-index/holder-case-store.cc new file mode 100644 index 00000000..ed8f9a2a Binary files /dev/null and b/tests/vectors/unique-index/holder-case-store.cc differ diff --git a/tests/vectors/unique-index/key-case-store.cc b/tests/vectors/unique-index/key-case-store.cc new file mode 100644 index 00000000..b5032c8d Binary files /dev/null and b/tests/vectors/unique-index/key-case-store.cc differ diff --git a/tests/vectors/unique-index/key-punctuation-store.cc b/tests/vectors/unique-index/key-punctuation-store.cc new file mode 100644 index 00000000..9f62d0be Binary files /dev/null and b/tests/vectors/unique-index/key-punctuation-store.cc differ diff --git a/tests/vectors/unique-index/normal-keys-store.cc b/tests/vectors/unique-index/normal-keys-store.cc new file mode 100644 index 00000000..27f36b96 Binary files /dev/null and b/tests/vectors/unique-index/normal-keys-store.cc differ diff --git a/tests/vectors/unique-index/normal-three-store.cc b/tests/vectors/unique-index/normal-three-store.cc new file mode 100644 index 00000000..580ae038 Binary files /dev/null and b/tests/vectors/unique-index/normal-three-store.cc differ diff --git a/tests/vectors/unique-index/outcomes.msgpack b/tests/vectors/unique-index/outcomes.msgpack new file mode 100644 index 00000000..2ff46011 --- /dev/null +++ b/tests/vectors/unique-index/outcomes.msgpack @@ -0,0 +1 @@ +“•²duplicate-store.cc£Sku§refused¦item:a¦item:b•²duplicate-store.ccÀ¤openÀÀ•®blank-store.cc£Sku¤openÀÀ \ No newline at end of file diff --git a/tests/vectors/unique-index/pair-store.cc b/tests/vectors/unique-index/pair-store.cc new file mode 100644 index 00000000..96a180ff Binary files /dev/null and b/tests/vectors/unique-index/pair-store.cc differ diff --git a/tests/vectors/unique-index/refusals.msgpack b/tests/vectors/unique-index/refusals.msgpack new file mode 100644 index 00000000..01dcc9e1 Binary files /dev/null and b/tests/vectors/unique-index/refusals.msgpack differ diff --git a/tests/vectors/unique-index/repull-after.cc b/tests/vectors/unique-index/repull-after.cc new file mode 100644 index 00000000..3d2a3a13 Binary files /dev/null and b/tests/vectors/unique-index/repull-after.cc differ diff --git a/tests/vectors/unique-index/repull-before.cc b/tests/vectors/unique-index/repull-before.cc new file mode 100644 index 00000000..8ee44ec0 Binary files /dev/null and b/tests/vectors/unique-index/repull-before.cc differ diff --git a/tests/vectors/unique-index/surrogate-groups-store.cc b/tests/vectors/unique-index/surrogate-groups-store.cc new file mode 100644 index 00000000..f9774c85 Binary files /dev/null and b/tests/vectors/unique-index/surrogate-groups-store.cc differ diff --git a/tests/vectors/unique-index/surrogate-store.cc b/tests/vectors/unique-index/surrogate-store.cc new file mode 100644 index 00000000..4d0a1daa Binary files /dev/null and b/tests/vectors/unique-index/surrogate-store.cc differ diff --git a/tests/vectors/unique-index/types-store.cc b/tests/vectors/unique-index/types-store.cc new file mode 100644 index 00000000..36c22d18 Binary files /dev/null and b/tests/vectors/unique-index/types-store.cc differ diff --git a/tests/vectors/unique-index/values.msgpack b/tests/vectors/unique-index/values.msgpack new file mode 100644 index 00000000..bbf93b0f Binary files /dev/null and b/tests/vectors/unique-index/values.msgpack differ diff --git a/tests/vectors/unique-index/wide-store.cc b/tests/vectors/unique-index/wide-store.cc new file mode 100644 index 00000000..2dcc1e4a Binary files /dev/null and b/tests/vectors/unique-index/wide-store.cc differ diff --git a/tools/sprite-silhouette/README.md b/tools/sprite-silhouette/README.md new file mode 100644 index 00000000..a959e193 --- /dev/null +++ b/tools/sprite-silhouette/README.md @@ -0,0 +1,68 @@ +# sprite-silhouette + +Turns a sprite's alpha channel into a simplified polygon in percent coordinates, for CSS +`shape-outside: polygon()`, so text wraps around the figure instead of its rectangle. Zero +dependencies: Node (tested on 24; `node:zlib` does the PNG inflate). Entry point: `silhouette.mjs`. +Live and tested; there is no packaging or release for it, it is a repo tool. + +## First call + +``` +node tools/sprite-silhouette/silhouette.mjs /opt/commons/cast-art/poses --out silhouettes.json +node --test tools/sprite-silhouette/silhouette.test.mjs +``` + +The first argument is a poses directory holding `catalogue.json` (the cast-art catalogue); the +tool reads it and every PNG it names, and writes nothing there. Output is a JSON object keyed by +pose id, `mind/label` in catalogue order (`void/neutral`, `eureka/probe`, ...): + +```json +{ "void/neutral": { "width": 605, "height": 461, + "points": [[42.98, 6.72], [49.59, 8.24], "..."], + "css": "polygon(42.98% 6.72%, 49.59% 8.24%, ...)" } } +``` + +Use the `css` string as the value of `shape-outside` on a floated image of the same aspect ratio +(percentages are of the image's box): + +```css +img.pose { float: left; shape-outside: polygon(42.98% 6.72%, 49.59% 8.24%, ...); shape-margin: 0.5em; } +``` + +An empty sprite (nothing at or above the threshold) gets `points: []` and `css: "none"`. + +From code: `silhouette(pngBuffer, { threshold, tolerance })` in `silhouette.mjs`. + +## Options + +- `--threshold 0.35`: a pixel is inside when `alpha/255 >= threshold` (0.35 is the Void mock-up's value). +- `--tolerance 1.5`: Douglas-Peucker distance in source pixels; the simplified polygon stays within + this of the traced edge in both directions (tested). 0 keeps every corner. +- `--out file`: write there instead of stdout. + +## What it does and does not trace + +Threshold, then the outer boundary of the largest 8-connected shape, by walking the pixel edges +(so vertices sit on pixel corners and the walk is exact), then simplification of that closed ring. +Holes are ignored (wrapping text goes around a figure, not through its gaps), and smaller +disconnected islands (a floating hand, a stray sparkle) are dropped: `shape-outside` takes one +polygon. The contour is a staircase of pixel edges, so below about 1 px the tolerance only +reproduces the staircase; there is no sub-pixel interpolation of the alpha ramp. + +Input: 8-bit non-interlaced PNG (RGBA, grey+alpha, palette with tRNS, or opaque RGB/grey, which is +a rectangle). Anything else is refused with its name. + +## Prior art + +- **Marching squares** (Lorensen and Cline's cubes, 2D case) interpolates along the alpha ramp for + sub-pixel contours and one pass emits every contour, holes included. Not used: the pose art is + anti-aliased over a pixel or two, a 1.5 px tolerance swallows that, and we want exactly one outer ring. +- **Potrace** (Selinger, 2003) fits optimal polygons and then Bezier curves to a bitmap. CSS + `polygon()` has no curves, so only its idea of an optimal polygon applies; this tool keeps the + simpler greedy Douglas-Peucker and measures its error directly. +- **Douglas-Peucker** (1973; Ramer 1972) is the simplifier: keep the vertex farthest from the chord + if it is beyond the tolerance, recurse. Its guarantee (every dropped vertex within the tolerance + of the result) is what the tests assert. It is not topology-preserving; on all 99 current poses at + the defaults no output self-intersects (checked when this was written, not asserted by the suite). +- **Boundary following** (Moore-neighbour / square tracing) is the contour walk, with the + 8-connected rule so diagonally touching pixels stay one shape. diff --git a/tools/sprite-silhouette/png.mjs b/tools/sprite-silhouette/png.mjs new file mode 100644 index 00000000..280e80ab --- /dev/null +++ b/tools/sprite-silhouette/png.mjs @@ -0,0 +1,82 @@ +// Minimal PNG reader: the alpha plane only. 8-bit, non-interlaced (every pose in cast-art is RGBA8). +// Anything else is refused by name, never guessed at. +import { inflateSync } from "node:zlib"; + +const SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const CHANNELS = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 }; + +/** @returns {{ width: number, height: number, alpha: Uint8Array }} alpha is row-major, 0..255. */ +export function decodeAlpha(buffer) { + if (buffer.length < 8 || !buffer.subarray(0, 8).equals(SIGNATURE)) throw new Error("not a PNG"); + let header = null; + let palette = null; + let trns = null; + const idat = []; + for (let at = 8; at + 8 <= buffer.length; ) { + const length = buffer.readUInt32BE(at); + const type = buffer.toString("latin1", at + 4, at + 8); + const body = buffer.subarray(at + 8, at + 8 + length); + if (body.length !== length) throw new Error(`truncated ${type} chunk`); + at += 12 + length; + if (type === "IHDR") { + header = { width: body.readUInt32BE(0), height: body.readUInt32BE(4), depth: body[8], color: body[9], interlace: body[12] }; + } else if (type === "PLTE") palette = body; + else if (type === "tRNS") trns = body; + else if (type === "IDAT") idat.push(body); + else if (type === "IEND") break; + } + if (!header) throw new Error("PNG has no IHDR"); + const { width, height, depth, color, interlace } = header; + if (depth !== 8) throw new Error(`unsupported PNG bit depth ${depth} (8 only)`); + if (interlace !== 0) throw new Error("unsupported PNG: interlaced"); + const channels = CHANNELS[color]; + if (!channels) throw new Error(`unsupported PNG color type ${color}`); + if (color === 3 && !palette) throw new Error("indexed PNG has no PLTE"); + + const raw = inflateSync(Buffer.concat(idat)); + const stride = width * channels; + if (raw.length !== (stride + 1) * height) throw new Error("PNG pixel data has the wrong length"); + + const alpha = new Uint8Array(width * height); + let prev = new Uint8Array(stride); + for (let y = 0; y < height; y++) { + const filter = raw[y * (stride + 1)]; + const line = Uint8Array.from(raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1))); + unfilter(filter, line, prev, channels); + for (let x = 0; x < width; x++) { + const p = x * channels; + let a = 255; + if (color === 6) a = line[p + 3]; + else if (color === 4) a = line[p + 1]; + else if (color === 3) a = trns && line[p] < trns.length ? trns[line[p]] : 255; + else if (trns && color === 0 && line[p] === trns.readUInt16BE(0)) a = 0; + else if (trns && color === 2 && line[p] === trns.readUInt16BE(0) && line[p + 1] === trns.readUInt16BE(2) && line[p + 2] === trns.readUInt16BE(4)) a = 0; + alpha[y * width + x] = a; + } + prev = line; + } + return { width, height, alpha }; +} + +function unfilter(filter, line, prev, bpp) { + for (let i = 0; i < line.length; i++) { + const a = i >= bpp ? line[i - bpp] : 0; + const b = prev[i]; + const c = i >= bpp ? prev[i - bpp] : 0; + let add; + switch (filter) { + case 0: add = 0; break; + case 1: add = a; break; + case 2: add = b; break; + case 3: add = (a + b) >> 1; break; + case 4: { + const p = a + b - c; + const pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + add = pa <= pb && pa <= pc ? a : pb <= pc ? b : c; + break; + } + default: throw new Error(`bad PNG filter ${filter}`); + } + line[i] = (line[i] + add) & 255; + } +} diff --git a/tools/sprite-silhouette/silhouette.mjs b/tools/sprite-silhouette/silhouette.mjs new file mode 100644 index 00000000..92cb56d7 --- /dev/null +++ b/tools/sprite-silhouette/silhouette.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Sprite alpha silhouette -> simplified polygon in percent coordinates for CSS shape-outside: polygon(). +// First call: node tools/sprite-silhouette/silhouette.mjs /opt/commons/cast-art/poses > silhouettes.json +import { readFileSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { decodeAlpha } from "./png.mjs"; +import { largestContour, mask, simplifyRing } from "./trace.mjs"; + +export const DEFAULTS = { threshold: 0.35, tolerance: 1.5 }; + +const pct = (v, total) => Math.round((v / total) * 10000) / 100; + +/** @returns {{ width: number, height: number, points: number[][], css: string }} points are [x%, y%]; css is "none" for an empty sprite. */ +export function silhouette(png, { threshold = DEFAULTS.threshold, tolerance = DEFAULTS.tolerance } = {}) { + const { width, height, alpha } = decodeAlpha(png); + const ring = largestContour(mask(alpha, threshold), width, height); + if (!ring) return { width, height, points: [], css: "none" }; + const points = simplifyRing(ring, tolerance).map(([x, y]) => [pct(x, width), pct(y, height)]); + const css = `polygon(${points.map(([x, y]) => `${x}% ${y}%`).join(", ")})`; + return { width, height, points, css }; +} + +/** Pose ids are `mind/label`, in catalogue order; the value is the path under the poses root. */ +export function poseFiles(catalogue) { + const files = {}; + for (const [mind, entry] of Object.entries(catalogue.minds)) { + for (const group of [entry.shared, entry.signatures]) { + for (const [label, path] of Object.entries(group ?? {})) files[`${mind}/${label}`] = path; + } + } + return files; +} + +export function silhouettes(root, options) { + const catalogue = JSON.parse(readFileSync(join(root, "catalogue.json"), "utf8")); + const out = {}; + for (const [id, path] of Object.entries(poseFiles(catalogue))) out[id] = silhouette(readFileSync(join(root, path)), options); + return out; +} + +function main(argv) { + const options = { ...DEFAULTS }; + let out = null, root = null; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--threshold") options.threshold = Number(argv[++i]); + else if (argv[i] === "--tolerance") options.tolerance = Number(argv[++i]); + else if (argv[i] === "--out") out = argv[++i]; + else if (argv[i].startsWith("--")) throw new Error(`unknown option ${argv[i]}`); + else root = argv[i]; + } + if (!root || !(options.threshold > 0 && options.threshold <= 1) || !(options.tolerance >= 0)) { + console.error("usage: silhouette.mjs [--threshold 0.35] [--tolerance 1.5] [--out file.json]"); + process.exit(2); + } + const text = JSON.stringify(silhouettes(root, options), null, 1) + "\n"; + if (out) writeFileSync(out, text); + else process.stdout.write(text); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) main(process.argv.slice(2)); diff --git a/tools/sprite-silhouette/silhouette.test.mjs b/tools/sprite-silhouette/silhouette.test.mjs new file mode 100644 index 00000000..c8b4faca --- /dev/null +++ b/tools/sprite-silhouette/silhouette.test.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import test from "node:test"; +import { deflateSync } from "node:zlib"; +import { decodeAlpha } from "./png.mjs"; +import { silhouette, silhouettes } from "./silhouette.mjs"; +import { largestContour, mask, maxDistanceToRing, simplifyRing } from "./trace.mjs"; + +// An RGBA8 PNG built from an alpha function, every row encoded with the given PNG filter. +function crc32(buf) { + let c, crc = ~0; + for (let i = 0; i < buf.length; i++) { + c = (crc ^ buf[i]) & 255; + for (let k = 0; k < 8; k++) c = c & 1 ? (c >>> 1) ^ 0xedb88320 : c >>> 1; + crc = (crc >>> 8) ^ c; + } + return ~crc >>> 0; +} +function chunk(type, body) { + const head = Buffer.alloc(8); + head.writeUInt32BE(body.length, 0); + head.write(type, 4, "latin1"); + const tail = Buffer.alloc(4); + tail.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), body])), 0); + return Buffer.concat([head, body, tail]); +} +function png(width, height, alphaAt, { filter = 0 } = {}) { + const rows = []; + let prev = Buffer.alloc(width * 4); + for (let y = 0; y < height; y++) { + const line = Buffer.alloc(width * 4); + for (let x = 0; x < width; x++) line.set([200, 10 * (x % 20), 30, alphaAt(x, y)], x * 4); + const out = Buffer.alloc(width * 4 + 1); + out[0] = filter; + for (let i = 0; i < line.length; i++) { + const a = i >= 4 ? line[i - 4] : 0, b = prev[i], c = i >= 4 ? prev[i - 4] : 0; + const p = a + b - c, pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c); + const pred = [0, a, b, (a + b) >> 1, pa <= pb && pa <= pc ? a : pb <= pc ? b : c][filter]; + out[i + 1] = (line[i] - pred) & 255; + } + rows.push(out); + prev = line; + } + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); ihdr.writeUInt32BE(height, 4); ihdr[8] = 8; ihdr[9] = 6; + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", ihdr), chunk("IDAT", deflateSync(Buffer.concat(rows))), chunk("IEND", Buffer.alloc(0)), + ]); +} +const disc = (cx, cy, r) => (x, y) => ((x + 0.5 - cx) ** 2 + (y + 0.5 - cy) ** 2 <= r * r ? 255 : 0); + +test("decodes alpha through every PNG row filter", () => { + const f = (x, y) => (x * 7 + y * 13) % 256; + for (const filter of [0, 1, 2, 3, 4]) { + const { width, height, alpha } = decodeAlpha(png(9, 7, f, { filter })); + assert.deepEqual([width, height], [9, 7]); + for (let y = 0; y < 7; y++) for (let x = 0; x < 9; x++) assert.equal(alpha[y * 9 + x], f(x, y), `filter ${filter} at ${x},${y}`); + } +}); + +test("refuses what it cannot read", () => { + assert.throws(() => decodeAlpha(Buffer.from("nope")), /not a PNG/); + const bad = png(2, 2, () => 255); + bad[24] = 16; // IHDR bit depth + assert.throws(() => decodeAlpha(bad), /bit depth 16/); +}); + +test("a fully transparent sprite has no shape", () => { + const s = silhouette(png(8, 8, () => 0)); + assert.deepEqual(s.points, []); + assert.equal(s.css, "none"); +}); + +test("a fully opaque sprite is its own bounding box", () => { + const s = silhouette(png(8, 4, () => 255)); + assert.deepEqual(s.points, [[0, 0], [100, 0], [100, 100], [0, 100]]); + assert.equal(s.css, "polygon(0% 0%, 100% 0%, 100% 100%, 0% 100%)"); +}); + +test("threshold is alpha/255 >= 0.35: 89 is out, 90 is in", () => { + assert.deepEqual(silhouette(png(4, 4, () => 89)).points, []); + assert.equal(silhouette(png(4, 4, () => 90)).points.length, 4); + assert.deepEqual(silhouette(png(4, 4, () => 89), { threshold: 0.3 }).points.length, 4); +}); + +test("simplified shape stays within tolerance of the traced edge, both ways, and is smaller", () => { + const { width, height, alpha } = decodeAlpha(png(120, 100, disc(60, 50, 40))); + const ring = largestContour(mask(alpha, 0.35), width, height); + for (const tolerance of [0.5, 1.5, 4]) { + const simple = simplifyRing(ring, tolerance); + assert.ok(simple.length < ring.length && simple.length >= 4); + assert.ok(maxDistanceToRing(ring, simple) <= tolerance + 1e-9, `ring to simplified at ${tolerance}`); + assert.ok(maxDistanceToRing(simple, ring) <= tolerance + 1e-9, `simplified to ring at ${tolerance}`); + } + // And the traced edge is the true edge to within a pixel: every ring vertex sits within 1px of the disc's circle. + for (const [x, y] of ring) assert.ok(Math.abs(Math.hypot(x - 60, y - 50) - 40) <= 1.0, `${x},${y}`); +}); + +test("only the outside is traced: a hole is ignored", () => { + const ring = (x, y) => (Math.max(Math.abs(x - 10), Math.abs(y - 10)) <= 8 && Math.max(Math.abs(x - 10), Math.abs(y - 10)) > 3 ? 255 : 0); + const s = silhouette(png(20, 20, ring), { tolerance: 0 }); + const xs = s.points.map((p) => p[0]), ys = s.points.map((p) => p[1]); + assert.deepEqual([Math.min(...xs), Math.max(...xs), Math.min(...ys), Math.max(...ys)], [10, 95, 10, 95]); + assert.equal(s.points.length, 4); +}); + +test("pixels that only touch diagonally are one shape; a smaller island is dropped", () => { + const dots = (x, y) => (x === y && x < 4 ? 255 : 0); + const diag = silhouette(png(8, 8, dots), { tolerance: 0 }); + const xs = diag.points.map((p) => p[0]); + assert.equal(Math.max(...xs), 50); // x 0..4 of 8 + const island = (x, y) => ((x < 5 && y < 5) || (x === 8 && y === 8) ? 255 : 0); + const s = silhouette(png(10, 10, island)); + assert.deepEqual(s.points, [[0, 0], [50, 0], [50, 50], [0, 50]]); +}); + +test("deterministic: same bytes, same polygon; coordinates are percent within the image", () => { + const bytes = png(90, 70, disc(45, 35, 30)); + const a = silhouette(bytes), b = silhouette(Buffer.from(bytes)); + assert.deepEqual(a, b); + for (const [x, y] of a.points) assert.ok(x >= 0 && x <= 100 && y >= 0 && y <= 100); + assert.match(a.css, /^polygon\((\d+(\.\d+)?% \d+(\.\d+)?%)(, \d+(\.\d+)?% \d+(\.\d+)?%){3,}\)$/); +}); + +const POSES = "/opt/commons/cast-art/poses"; +test("every pose in the shared cast-art catalogue gets a polygon", { skip: !existsSync(`${POSES}/catalogue.json`) }, () => { + const all = silhouettes(POSES); + const ids = Object.keys(all); + assert.ok(ids.length > 90 && ids.includes("void/neutral")); + for (const id of ids) assert.ok(all[id].points.length >= 4 && all[id].points.length < 400, id); + assert.deepEqual(silhouettes(POSES), all); +}); diff --git a/tools/sprite-silhouette/trace.mjs b/tools/sprite-silhouette/trace.mjs new file mode 100644 index 00000000..49594a5b --- /dev/null +++ b/tools/sprite-silhouette/trace.mjs @@ -0,0 +1,109 @@ +// Silhouette of an alpha plane: threshold -> outer contour of the largest 8-connected shape -> Douglas-Peucker. +// Coordinates are pixel-corner lattice points: x in 0..width, y in 0..height. + +/** A pixel is inside when alpha/255 >= threshold. */ +export function mask(alpha, threshold) { + const m = new Uint8Array(alpha.length); + for (let i = 0; i < alpha.length; i++) m[i] = alpha[i] / 255 >= threshold ? 1 : 0; + return m; +} + +/** Outer contour (lattice vertices at turns, clockwise on screen) of the 8-connected component with the most pixels, or null. */ +export function largestContour(m, width, height) { + const seen = new Uint8Array(m.length); + let best = null; + for (let start = 0; start < m.length; start++) { + if (!m[start] || seen[start]) continue; + // Raster order: `start` is the component's first pixel, so the pixels above and up-left of it are empty + // or belong to another component. + let count = 0; + const stack = [start]; + seen[start] = 1; + while (stack.length) { + const p = stack.pop(); + count++; + const px = p % width, py = (p / width) | 0; + for (let dy = -1; dy <= 1; dy++) for (let dx = -1; dx <= 1; dx++) { + const x = px + dx, y = py + dy; + if (x < 0 || y < 0 || x >= width || y >= height) continue; + const q = y * width + x; + if (m[q] && !seen[q]) { seen[q] = 1; stack.push(q); } + } + } + if (!best || count > best.count) best = { count, start }; + } + return best && traceOuter(m, width, height, best.start % width, (best.start / width) | 0); +} + +// Directions: 0 east, 1 south, 2 west, 3 north. The shape is kept on the walker's right, so an +// outer boundary comes out clockwise; at each corner a filled left-ahead pixel turns left (which +// joins pixels that only touch diagonally), else a filled right-ahead pixel goes straight, else right. +const DX = [1, 0, -1, 0], DY = [0, 1, 0, -1]; +const AHEAD_LEFT = [[0, -1], [0, 0], [-1, 0], [-1, -1]]; +const AHEAD_RIGHT = [[0, 0], [-1, 0], [-1, -1], [0, -1]]; + +function traceOuter(m, width, height, px, py) { + const at = (x, y) => x >= 0 && y >= 0 && x < width && y < height && m[y * width + x] === 1; + const vertices = []; + let x = px, y = py, dir = 3, first = true; // arrive at the start corner heading north, so it is counted as a turn + for (;;) { + const left = at(x + AHEAD_LEFT[dir][0], y + AHEAD_LEFT[dir][1]); + const right = at(x + AHEAD_RIGHT[dir][0], y + AHEAD_RIGHT[dir][1]); + const turn = left ? 3 : right ? 0 : 1; + const next = (dir + turn) & 3; + // Back at the start corner and about to leave east: the ring is closed. + if (!first && x === px && y === py && next === 0) break; + first = false; + if (turn !== 0) vertices.push([x, y]); + dir = next; + x += DX[dir]; y += DY[dir]; + } + return vertices; +} + +/** Douglas-Peucker on a closed ring: split at the vertex farthest from the first, simplify both halves. */ +export function simplifyRing(ring, tolerance) { + if (ring.length <= 3 || tolerance <= 0) return ring.slice(); + let far = 0, best = -1; + for (let i = 1; i < ring.length; i++) { + const d = (ring[i][0] - ring[0][0]) ** 2 + (ring[i][1] - ring[0][1]) ** 2; + if (d > best) { best = d; far = i; } + } + const a = dp(ring.slice(0, far + 1), tolerance); + const b = dp(ring.slice(far).concat([ring[0]]), tolerance); + return a.slice(0, -1).concat(b.slice(0, -1)); +} + +function dp(points, tolerance) { + const keep = new Uint8Array(points.length); + keep[0] = keep[points.length - 1] = 1; + const stack = [[0, points.length - 1]]; + while (stack.length) { + const [lo, hi] = stack.pop(); + let worst = -1, at = -1; + for (let i = lo + 1; i < hi; i++) { + const d = distanceToSegment(points[i], points[lo], points[hi]); + if (d > worst) { worst = d; at = i; } + } + if (worst > tolerance) { keep[at] = 1; stack.push([lo, at], [at, hi]); } + } + return points.filter((_, i) => keep[i]); +} + +export function distanceToSegment(p, a, b) { + const vx = b[0] - a[0], vy = b[1] - a[1]; + const len2 = vx * vx + vy * vy; + const t = len2 === 0 ? 0 : Math.max(0, Math.min(1, ((p[0] - a[0]) * vx + (p[1] - a[1]) * vy) / len2)); + return Math.hypot(p[0] - (a[0] + t * vx), p[1] - (a[1] + t * vy)); +} + +/** Largest distance from any point of ring `a`'s edges' vertices to ring `b`'s edges. */ +export function maxDistanceToRing(points, ring) { + let worst = 0; + for (const p of points) { + let d = Infinity; + for (let i = 0; i < ring.length; i++) d = Math.min(d, distanceToSegment(p, ring[i], ring[(i + 1) % ring.length])); + worst = Math.max(worst, d); + } + return worst; +} diff --git a/unity/org.gamecult.cultlib/CHANGELOG.md b/unity/org.gamecult.cultlib/CHANGELOG.md index ef98338b..b5ff6419 100644 --- a/unity/org.gamecult.cultlib/CHANGELOG.md +++ b/unity/org.gamecult.cultlib/CHANGELOG.md @@ -2,15 +2,13 @@ All notable changes to this package are documented in this file. -## [Unreleased] +## [3.0.0] - 2026-11-11 -Releases as 3.0.0 (major). - -### Breaking (major) +### Breaking - A single-file store flush refuses a file it cannot read (empty, truncated, an unreadable stamp or body) with `CultStoreUnreadableException` and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. - A single-file store reads v1, v2 and v3 stamps and refuses v4 and later by name; a directory store reads v4 and v5 and refuses v6 and later by name. Each judges the stamp before the body, so an unreadable body never makes a newer store look like damage. -- A store is written in the format its content needs, and CultLib 2.0 refuses the newer ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. +- A store is written in the format its content needs, and CultLib 2.0 refuses the newer single-file ones by name. A single-file store holding a document variant is written as `cultcache.store.v2`, one holding an element id as `cultcache.store.v3`; a directory store holding an element id is written as `cultcache.store.v5` (`v4` otherwise). A store is marked only when a record actually holds an id or a variant. Upgrade readers before writers: a 2.0 reader that opens such a store is refused, not misread. Rolling back to a 2.0 pin after a store has been written as v3 or v5 is one-way: the only carry-back is narrow and lossy (a whole-store write that leaves no element holding an id writes the store unmarked, and the ids are gone), so restore from a copy taken before upgrading. Refusal by name is not uniform in 1.x: a v5 directory store is refused from `cultlib-unity-v1.0.57` (`caching-unity-v1.2.0`), but a v3 single-file store is read silently by every pin before `cultlib-unity-v2.0.0` (including `cultlib-unity-v1.0.61`, `caching-unity-v1.4.0`, `cultlib-v1.0.32`), which drops the ids and can rewrite the elements without them. - Writes are write sets. A flush or commit replaces the records it staged and copies every other record forward as stored, under the store's lock, instead of rewriting the whole store from the cache's memory. A record the cache does not know is kept, and an untouched record keeps a dropped member's slot through every write (never reuse a dropped slot; see the schema-compatibility contract). - A write or removal at a key whose stored record the writer does not own is refused with `CultSchemaConflictException`, and the file is left as it was. A writer owns a record whose schema id it declares (owns or lists compatible) and a record its type reads whole under compatible drift: that record is writable and is restamped under the type's own id on write, as in 2.0.1. A record a type reads only by dropping persisted slots stays read-only until its id is declared compatible, which accepts the loss (this differs from 2.0.1, which shed the slots silently). A cache owns through its registry; a cacheless CultMesh write owns through the catalog entry it writes. - A write replaces the record at a key only when the type it writes there owns that record, decided per key and the same for a cache and for a cacheless `WriteDocument`, `WriteRecord` or unattached-store `PushAll`, in a single-file store and a directory store. The written type owns the stored record when its entry declares the stored id (its own id, or one it lists compatible), or when it is the type that reads the record's published entry and reads it whole (no persisted slot unread): an older id of its own schema name that it reads whole is its own and is restamped, as in 2.0.1. Anything else is refused with `CultSchemaConflictException` and the file left as it was: a record of another registered type, under its current id or an old id it lists, and a record the written type would read only by dropping persisted slots. No registry enters the decision, `CultDocumentRegistry.Shared` included, so a lent registry never changes the answer. This changes a cache: where 2.x let a cache holding two types replace one's record at a key with the other's, it now refuses, with or without a backing store. Retype a key by removing it and adding the new record, in one batch (`batch.Remove(key)` then `batch.Upsert(...)`) or staged apart (`Remove` then `UpsertAsync`); the removal is judged as a removal, so a record no registered type owns is still refused. A removal or check goes to the registry's removal rule, and a cacheless call, which has no registry, removes a record only when an entry it writes owns it (a `Delete` then `PushAll` on an unattached store writes no entry and is refused). diff --git a/unity/org.gamecult.cultlib/README.md b/unity/org.gamecult.cultlib/README.md index e118ac65..30db7fba 100644 --- a/unity/org.gamecult.cultlib/README.md +++ b/unity/org.gamecult.cultlib/README.md @@ -11,19 +11,19 @@ and OpenSSL license notices are included under `Third Party Notices`. Build it with `scripts/build-unity-package.ps1`. Do not edit generated plugin assemblies in the package output. -Install version `2.0.0` through Unity Package Manager with: +Install version `3.0.0` through Unity Package Manager with: ```text -https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v2.0.0 +https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v3.0.0 ``` -The package depends on `org.gamecult.cultmath` (exactly 0.5.0) and does not +The package depends on `org.gamecult.cultmath` (exactly 0.5.1) and does not carry its own `CultMath.dll`. Unity Package Manager resolves git packages only from the project manifest, so add CultMath beside CultLib in `Packages/manifest.json`: ```text -"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath#cultmath-unity-v0.5.0" +"org.gamecult.cultmath": "https://github.com/GameCult/CultLib.git?path=/packages/cultmath/unity/org.gamecult.cultmath#cultmath-unity-v0.5.1" ``` CultMesh chooses mapped memory for reachable same-machine bodies and advertised diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll index bf05fec2..5b040bec 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb index bfe11807..d2629dfd 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.MessagePack.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll index 8e7b88f1..cff0d028 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb index 3fb019e1..d952c867 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Caching.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll index 2dd55355..f9674828 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb index 1635f093..f881acde 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Logging.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll index 6e138fd5..9bff9912 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb index 021de32d..62bdb6b3 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.Quic.Native.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll index ceb2fbf4..f7a86dde 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb index 3074fbd6..d27197b5 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Mesh.pdb differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll index 21e64ce1..da269490 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.WebSockets.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll index dd6482f9..065ac7c5 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.dll differ diff --git a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb index 4fcf53c6..8cb5dad7 100644 Binary files a/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb and b/unity/org.gamecult.cultlib/Runtime/Plugins/GameCult.Networking.pdb differ diff --git a/unity/org.gamecult.cultlib/package.json b/unity/org.gamecult.cultlib/package.json index 19e42b58..1111d3fb 100644 --- a/unity/org.gamecult.cultlib/package.json +++ b/unity/org.gamecult.cultlib/package.json @@ -1,10 +1,10 @@ { "name": "org.gamecult.cultlib", "displayName": "CultLib", - "version": "2.0.1", + "version": "3.0.0", "unity": "2021.3", "dependencies": { - "org.gamecult.cultmath": "0.5.0" + "org.gamecult.cultmath": "0.5.1" }, "description": "CultCache, CultNet, and CultMesh managed runtime assemblies for Unity.", "license": "MIT",