Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 892765ba97 | |||
| 5e10909fe9 | |||
| a8c2825745 | |||
| c371832736 | |||
| a0184785a2 | |||
| 7a63290469 | |||
| 6d1684e1c2 | |||
| 871d81807b | |||
| d3025d3c19 | |||
| 3a51bbabad | |||
| 19b95ba2ab | |||
| af9142539e | |||
| f4eb2a04d1 | |||
| 8beb972c68 | |||
| 5fa7057ed1 | |||
| 0ebc7d63e8 | |||
| 4246a37284 | |||
| b88c0a8ee2 | |||
| f2bb0dd1dd | |||
| 7c2e514941 | |||
| 17caa0c874 | |||
| 1211e8c6dc | |||
| 3deb270458 | |||
| 7225835d11 | |||
| f1edfa458d | |||
| a57397aa7f | |||
|
|
8eb77546eb | ||
|
|
6e0e2feb74 | ||
|
|
3cb5a1753a | ||
| 063835f679 | |||
|
|
8adf014c66 | ||
| 23d254a501 | |||
| 7002c252ed | |||
| a6861adadb | |||
| 5e610b1b6e | |||
| 447b310b4d | |||
| 43aa451fb9 | |||
| a7c22e6ee5 | |||
| ba25b43369 |
161 changed files with 14399 additions and 2002 deletions
63
.github/workflows/cultmesh-portability.yml
vendored
63
.github/workflows/cultmesh-portability.yml
vendored
|
|
@ -1,11 +1,18 @@
|
||||||
name: CultMesh portability and performance
|
name: CultMesh portability and performance
|
||||||
|
|
||||||
|
# The forge runs one job at a time and this workflow holds the slot 14 to 30 minutes, so it runs once per change:
|
||||||
|
# on the pull request (never again on the push that merges it; a break that merges from a stale base waits for the
|
||||||
|
# schedule), skipped for design-note-only changes, and on a
|
||||||
|
# daily schedule so main is still exercised when nobody opens a pull request.
|
||||||
on:
|
on:
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
- "codex/**"
|
|
||||||
pull_request:
|
pull_request:
|
||||||
|
paths-ignore:
|
||||||
|
# Not `**/*.md` or `docs/**`: docs/start/README.md is read by the first-call tests of every runtime.
|
||||||
|
- "docs/*.md"
|
||||||
|
- "research/**"
|
||||||
|
- ".voidbot/**"
|
||||||
|
schedule:
|
||||||
|
- cron: "17 3 * * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
|
|
@ -18,7 +25,9 @@ jobs:
|
||||||
reactive-and-resource-lifetime:
|
reactive-and-resource-lifetime:
|
||||||
name: ${{ matrix.os }} reactive semantics and leases
|
name: ${{ matrix.os }} reactive semantics and leases
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
timeout-minutes: 30
|
# The job dies at the lower of this value and the forge runner's own cap, which Eureka is raising from 30 to 60 minutes
|
||||||
|
# (run 1378 carried 60 and was still cancelled at 30:02); the two must both say 60 before a full run can finish.
|
||||||
|
timeout-minutes: 60
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
|
|
@ -38,17 +47,41 @@ jobs:
|
||||||
with:
|
with:
|
||||||
dotnet-version: 10.0.x
|
dotnet-version: 10.0.x
|
||||||
|
|
||||||
|
# The forge runner image (catthehacker/ubuntu:act-latest) carries Node 24 and Python 3.12 on PATH, and these two
|
||||||
|
# setup actions downloaded them again from github.com anyway (about 2.7 and 2.8 minutes, run 639) because the
|
||||||
|
# image's tool cache is not the one they read. They run only where the image does not carry the tools (the
|
||||||
|
# workflow_dispatch Windows leg); the check below fails a Linux image that stops carrying them.
|
||||||
- name: Set up Node.js
|
- name: Set up Node.js
|
||||||
|
if: runner.os != 'Linux'
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@v4
|
||||||
with:
|
with:
|
||||||
node-version: 24
|
node-version: 24
|
||||||
cache: npm
|
cache: npm
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
|
if: runner.os != 'Linux'
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: "3.12"
|
python-version: "3.12"
|
||||||
|
|
||||||
|
- name: Check the image carries Node 24 and Python 3.12
|
||||||
|
if: runner.os == 'Linux'
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
node --version | grep -E '^v24\.'
|
||||||
|
python --version | grep -E '^Python 3\.12\.'
|
||||||
|
|
||||||
|
# The image's Python is distro-managed (PEP 668): pip refuses to install into it, so the Linux leg installs the
|
||||||
|
# CultLib packages into a venv under RUNNER_TEMP that later steps find first on PATH.
|
||||||
|
- name: Create the Python environment
|
||||||
|
if: runner.os == 'Linux'
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python -m venv "$RUNNER_TEMP/venv"
|
||||||
|
echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Install TypeScript workspace dependencies
|
- name: Install TypeScript workspace dependencies
|
||||||
run: npm install --no-package-lock
|
run: npm install --no-package-lock
|
||||||
|
|
||||||
|
|
@ -115,6 +148,26 @@ jobs:
|
||||||
- name: Verify the CultCache document owner with the TypeScript cache
|
- name: Verify the CultCache document owner with the TypeScript cache
|
||||||
run: npm run test --workspace packages/cultcache-ts
|
run: npm run test --workspace packages/cultcache-ts
|
||||||
|
|
||||||
|
# The Python cache's unit suite ran only in cultnet-interop.yml (windows-latest, dispatch-only), so on Linux
|
||||||
|
# it ran nowhere. It sits after the TypeScript cache's test so the built cultcache-ts store holder exists for the
|
||||||
|
# cross-runtime store-lock test, which skips without it.
|
||||||
|
- name: Verify the CultCache document owner with the Python cache
|
||||||
|
run: python -m unittest discover -s packages/cultcache-py/tests
|
||||||
|
|
||||||
|
# cargo test for packages/cultcache-rs ran in no workflow: the Rust cache's own unit and doc tests were unwatched
|
||||||
|
# (only the interop example built, inside the C# store tests). The target directory is under RUNNER_TEMP, away from the checkout.
|
||||||
|
- name: Verify the CultCache document owner with the Rust cache
|
||||||
|
shell: bash
|
||||||
|
run: CARGO_TARGET_DIR="$RUNNER_TEMP/cultcache-rs-target" cargo test --manifest-path packages/cultcache-rs/Cargo.toml
|
||||||
|
|
||||||
|
# tests/GameCult.Caching.Tests, the C# reference cache's own suite (about 1,260 tests, under a minute), ran in no
|
||||||
|
# forge workflow; only the Mesh tests above exercised the reference.
|
||||||
|
# The forge runner is globalization-invariant, so ADeterministicIdIsTheSameUnderEveryCulture... cannot construct a
|
||||||
|
# culture and throws CultureNotFoundException; it is excluded here and still runs wherever ICU is present
|
||||||
|
# (variants:follow_up:culture-test-invariant-image).
|
||||||
|
- name: Verify the CultCache document owner with the C# reference
|
||||||
|
run: dotnet test tests/GameCult.Caching.Tests/GameCult.Caching.Tests.csproj --filter "FullyQualifiedName!~ADeterministicIdIsTheSameUnderEveryCulture" --logger "console;verbosity=minimal"
|
||||||
|
|
||||||
# Packs the four TypeScript packages and installs them into a clean consumer, so a package that builds but does
|
# Packs the four TypeScript packages and installs them into a clean consumer, so a package that builds but does
|
||||||
# not install or load fails here.
|
# not install or load fails here.
|
||||||
- name: Verify the packed TypeScript packages install and load from a clean consumer
|
- name: Verify the packed TypeScript packages install and load from a clean consumer
|
||||||
|
|
|
||||||
8
.github/workflows/quic-native.yml
vendored
8
.github/workflows/quic-native.yml
vendored
|
|
@ -9,13 +9,21 @@ name: CultMesh QUIC native
|
||||||
# from the pinned debian:13 container below, never from a local or Yggdrasil
|
# from the pinned debian:13 container below, never from a local or Yggdrasil
|
||||||
# build.
|
# build.
|
||||||
|
|
||||||
|
# Only a change to the committed binaries, their manifest or this workflow can change what verify-committed checks,
|
||||||
|
# so a branch push or pull request runs it for those paths alone; a tag always runs it.
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
paths:
|
||||||
|
- "packages/cultmesh-ts/native/**"
|
||||||
|
- ".github/workflows/quic-native.yml"
|
||||||
tags:
|
tags:
|
||||||
- "cultmesh-quic-native-v*"
|
- "cultmesh-quic-native-v*"
|
||||||
pull_request:
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- "packages/cultmesh-ts/native/**"
|
||||||
|
- ".github/workflows/quic-native.yml"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
|
|
|
||||||
|
|
@ -899,23 +899,23 @@
|
||||||
{
|
{
|
||||||
"label": "trailing_nil",
|
"label": "trailing_nil",
|
||||||
"inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsA=",
|
"inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsA=",
|
||||||
"accepted": true,
|
"accepted": false,
|
||||||
"reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=",
|
"reencodedBase64": null,
|
||||||
"recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
"recordKey": null
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "trailing_reserved_code",
|
"label": "trailing_reserved_code",
|
||||||
"inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsE=",
|
"inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdsE=",
|
||||||
"accepted": true,
|
"accepted": false,
|
||||||
"reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=",
|
"reencodedBase64": null,
|
||||||
"recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
"recordKey": null
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "trailing_truncated_array",
|
"label": "trailing_truncated_array",
|
||||||
"inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpI=",
|
"inputBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRlNlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWuRoXiBoWuhdpI=",
|
||||||
"accepted": true,
|
"accepted": false,
|
||||||
"reencodedBase64": "mqNhL2KlYXNzZXShMdlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQCvYXBwbGljYXRpb24veC10oXSRldlAZTNiMGM0NDI5OGZjMWMxNDlhZmJmNGM4OTk2ZmI5MjQyN2FlNDFlNDY0OWI5MzRjYTQ5NTk5MWI3ODUyYjg1NQAAoWugkaF4gaFroXY=",
|
"reencodedBase64": null,
|
||||||
"recordKey": "mesh:cdn:artifact:a-b:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
"recordKey": null
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "top_nil",
|
"label": "top_nil",
|
||||||
|
|
|
||||||
|
|
@ -146,6 +146,9 @@ A malformed put or delete is `message_invalid`, not a per-operation code: the pe
|
||||||
commit's changes as one commit, so the replica judges the result (a swap of two unique-index values) and not the state
|
commit's changes as one commit, so the replica judges the result (a swap of two unique-index values) and not the state
|
||||||
between records. A replica whose cache refuses a commit is faulted, never silent. `commitId` is opaque and valid only
|
between records. A replica whose cache refuses a commit is faulted, never silent. `commitId` is opaque and valid only
|
||||||
within the subscription. The C# subscription server and database server tag commits; other runtimes adopt next.
|
within the subscription. The C# subscription server and database server tag commits; other runtimes adopt next.
|
||||||
|
- The C# writer sets every key of a `database_change_raw` message, nil where unset: a removed change carries `document`
|
||||||
|
nil, an added or updated one `recordKey` and `schemaId` nil. The schema accepts those nils; an added or updated change
|
||||||
|
still needs a document, and a removed one a `recordKey` and `schemaId`.
|
||||||
- Client-owned input documents may be predicted locally only inside explicit
|
- Client-owned input documents may be predicted locally only inside explicit
|
||||||
`CultNetClientAuthorityScope` declarations.
|
`CultNetClientAuthorityScope` declarations.
|
||||||
- Predicted input state is not authoritative shared state. The shard log still
|
- Predicted input state is not authoritative shared state. The shard log still
|
||||||
|
|
|
||||||
|
|
@ -10,17 +10,29 @@
|
||||||
"messageId": { "type": "string", "minLength": 1 },
|
"messageId": { "type": "string", "minLength": 1 },
|
||||||
"subscriptionId": { "type": "string", "minLength": 1 },
|
"subscriptionId": { "type": "string", "minLength": 1 },
|
||||||
"changeKind": { "enum": ["added", "updated", "removed"] },
|
"changeKind": { "enum": ["added", "updated", "removed"] },
|
||||||
"document": { "$ref": "./cultnet.raw-document-record.schema.json" },
|
"document": {
|
||||||
"recordKey": { "type": "string", "minLength": 1 },
|
"oneOf": [{ "type": "null" }, { "$ref": "./cultnet.raw-document-record.schema.json" }]
|
||||||
"schemaId": { "type": "string", "minLength": 1 },
|
},
|
||||||
|
"recordKey": { "type": ["string", "null"], "minLength": 1 },
|
||||||
|
"schemaId": { "type": ["string", "null"], "minLength": 1 },
|
||||||
"commitId": { "type": ["string", "null"], "minLength": 1 },
|
"commitId": { "type": ["string", "null"], "minLength": 1 },
|
||||||
"commitCount": { "type": ["integer", "null"], "minimum": 2 }
|
"commitCount": { "type": ["integer", "null"], "minimum": 2 }
|
||||||
},
|
},
|
||||||
"allOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
"if": { "properties": { "changeKind": { "const": "removed" } } },
|
"properties": {
|
||||||
"then": { "required": ["recordKey", "schemaId"] },
|
"changeKind": { "const": "removed" },
|
||||||
"else": { "required": ["document"] }
|
"recordKey": { "type": "string" },
|
||||||
|
"schemaId": { "type": "string" }
|
||||||
|
},
|
||||||
|
"required": ["recordKey", "schemaId"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"changeKind": { "enum": ["added", "updated"] },
|
||||||
|
"document": { "$ref": "./cultnet.raw-document-record.schema.json" }
|
||||||
|
},
|
||||||
|
"required": ["document"]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -26,13 +26,15 @@ All documents are CultNet map-mode: every key written, nil where absent, closed
|
||||||
|
|
||||||
### card
|
### card
|
||||||
|
|
||||||
`{schemaVersion, fork_id, title, context, options[{id, text}], recommendation, depends[], leans, flag, raised_in, deciders[], status, ruling, ruled_from[], order, options_hash}`.
|
`{schemaVersion, fork_id, title, context, options[{id, text}], recommendation, depends[], leans, flag, raised_in, raised_at, deciders[], decider_count, status, ruling, ruled_from[], order, options_hash}`.
|
||||||
|
|
||||||
- `fork_id`: the mind's `question` key, `<campaign>:question:<label>`.
|
- `fork_id`: the mind's `question` key, `<campaign>:question:<label>`.
|
||||||
- **Hashed (the shown members):** `title`, `context`, `options` (each `{id, text}`, in shown order), `recommendation` (an option id, **never nil**: the stored `question` requires `recommended`), `depends` (strings, each one of the stored question's `depends` lines verbatim, at most 8, each at most 1000 UTF-8 bytes (the stored `Line`), in shown order, empty when none; the stored kind holds prose lines, so a line may read like a reference such as `campaign:kind:label` but is hashed as the text it is).
|
- **Hashed (the shown members):** `title`, `context`, `options` (each `{id, text}`, in shown order), `recommendation` (an option id, **never nil**: the stored `question` requires `recommended`), `depends` (strings, each one of the stored question's `depends` lines verbatim, at most 8, each at most 1000 UTF-8 bytes (the stored `Line`), in shown order, empty when none; the stored kind holds prose lines, so a line may read like a reference such as `campaign:kind:label` but is hashed as the text it is).
|
||||||
- **Not hashed:** `fork_id` (bound beside the hash), `leans` (an option id or nil), `flag`, `raised_in`, `deciders`, `status` (`open` or `closed`), `ruling` (the ruling id when a ruling closed the fork; nil when the card is open, and nil on a closed card whose question was withdrawn or otherwise resolved with no ruling), `ruled_from` (answer ids a ruling has been admitted from; non-empty only on a closed card), `order`, and `options_hash` itself.
|
- **Not hashed:** `fork_id` (bound beside the hash), `leans` (an option id or nil), `flag`, `raised_in`, `raised_at` (the instant the question was first admitted, an RFC3339 instant string as `answered_at`, or nil when the mind holds no admission time, as for a card projected before the field; the key is always written; it never moves when the question is edited; age is the client's: now minus `raised_at`), `deciders`, `decider_count` (the number of deciders of the call, set beside `deciders`; neither it nor the emptiness of `deciders` enters the hash, so the hash a non-decider recomputes is a decider's), `status` (`open` or `closed`), `ruling` (the ruling id when a ruling closed the fork; nil when the card is open, and nil on a closed card whose question was withdrawn or otherwise resolved with no ruling), `ruled_from` (answer ids a ruling has been admitted from; non-empty only on a closed card), `order`, and `options_hash` itself.
|
||||||
- **`deciders` is read-only data.** The list is set outside the Mind, through Bifrost's admin path, and **Bifrost sends it per call**: Huginn keeps no copy (M5). The card projection copies the list the call carried. **The list holds one to 64 ids** (Huginn's bound), on the card and in the per-call list alike (vectors `deciders_at_the_cap_of_64`, `deciders_65`); the per-call list has no schema here, so the bound is stated for the cut that builds it. A question's own `audience` may only narrow it. A Mind never writes it, so a Mind cannot add itself and answer its own fork.
|
- **`deciders` is read-only data.** The list is set outside the Mind, through Bifrost's admin path, and **Bifrost sends it per call**: Huginn keeps no copy (M5). The card projection copies the list the call carried. **The per-call list holds one to 64 ids** (Huginn's bound; vectors `deciders_at_the_cap_of_64`, `deciders_65`); the per-call list has no schema here, so the bound is stated for the cut that builds it. **`decider_count` is the size of that list, an integer 1 to 64, always written** (vectors `decider_count_missing`, `_zero`, `_65`, `_string`, `_null`). A fork with no decider is still refused, now as `decider_count` 0. **`deciders` holds that list for a decider and is empty for a reader who is not one: empty means "the ids are withheld from this reader", never "no deciders"** (vector `withheld_deciders`). Rule R-card: when `deciders` is non-empty it holds exactly `decider_count` ids; a visible list is whole (vectors `decider_count_fewer_than_the_ids`, `decider_count_more_than_the_ids_shown`). A question's own `audience` may only narrow it. A Mind never writes it, so a Mind cannot add itself and answer its own fork.
|
||||||
- Rules a schema cannot say, each with a negative vector: a `depends` line is at most 1000 UTF-8 bytes (the schema bounds 8 lines and 1000 characters; bytes need the rule); option ids are unique; `order` is unique among a Mind's cards (a list with two cards on one `order` is refused, naming both fork ids); `recommendation` and `leans` name an option; deciders are unique; an open card names no `ruling` and its `ruled_from` is empty (a ruling that answers a fork closes it); `ruled_from` is non-empty only when `ruling` is set; a closed card may name no ruling (the question was withdrawn, superseded or deferred: step 6 reads `status`, not the ruling); `options_hash` equals the hash of the shown members.
|
- Rules a schema cannot say, each with a negative vector: a `depends` line is at most 1000 UTF-8 bytes (the schema bounds 8 lines and 1000 characters; bytes need the rule); option ids are unique; `order` is unique among a Mind's cards (a list with two cards on one `order` is refused, naming both fork ids); `recommendation` and `leans` name an option; deciders are unique; an open card names no `ruling` and its `ruled_from` is empty (a ruling that answers a fork closes it); `ruled_from` is non-empty only when `ruling` is set; a closed card may name no ruling (the question was withdrawn, superseded or deferred: step 6 reads `status`, not the ruling); `options_hash` equals the hash of the shown members; R-card (above); R-answer-result: `answer_result.answer.answerer` is never nil, being the sender's own stamped account (vector `answer_result_with_a_nil_answerer`); R-record, R-entry and R-list ("Reader views" below).
|
||||||
|
|
||||||
|
**Reader views.** A card and its record are shown in one of two views. The **decider view** carries the ids: `deciders` whole, every `answerer` named, and the counts equal to the ids. The **non-decider view** withholds them: `deciders` empty, every `answerer` nil, the counts kept (`decider_count`; `answerer_count` on the record). Who is a decider is Huginn's decision, not this contract's: a reader is a decider exactly when the call's subject is byte-equal to one of the call's deciders (`eureka-substrate:ruling:desk-read-visibility-withheld-ids`); Huginn's door withholds, Bifrost cannot (the card is opaque to it). There is no viewer flag on any document: the view is read off whether `deciders` is empty. This contract carries both views and refuses a view that leaks: R-record: if every `answerer` is named, `answerer_count` is the number of distinct ids; if all are nil and there is at least one answer, `answerer_count` is between 1 and the number of answers; with no answers it is 0 (vectors `answerer_count_disagrees_with_the_ids`, `answerer_count_counts_answers_not_people`, `withheld_count_over_the_answers`, `withheld_count_zero_over_answers`, `withheld_count_over_no_answers`); R-entry, the leak rule: an entry with `deciders` non-empty names every answerer, an entry with `deciders` empty names none (vectors `decider_card_with_withheld_answers`, `withheld_card_with_named_answers`, `withheld_card_with_a_named_answerer_among_nils`); R-list: all entries of one page agree on the view (vector `a_page_mixes_views`). The view does not move the page: `non_decider_page` has the counts, order, cursor and stamp of the decider page it twins, and the vectors' no-leak check finds no id string in a non-decider vector's bytes. `options_hash` is the same in both views, and the bound-answer and `stale_fork` machinery is untouched.
|
||||||
|
|
||||||
### Options hash (canonical bytes)
|
### Options hash (canonical bytes)
|
||||||
|
|
||||||
|
|
@ -65,7 +67,9 @@ The stamped `authority` field of a stored answer is an **attribution**: the role
|
||||||
|
|
||||||
`received_at` is Huginn's admission time; "first" means first by admission order (the `n<seq>` ordinal), never by `answered_at`. `claim_in_future` is true when `answered_at` is more than five minutes after `received_at`: accepted but flagged. It is a function of the two stored values, so the writer compares against the `received_at` it stores, at the precision it stores it (a flag decided against an unrounded clock and a stored second-truncated `received_at` disagrees with the document it sits in). **Not stored, and no answer id:** a body refused at step 1 or 2 (Bifrost; it never reaches Huginn), and a body naming an unknown fork (step 3; Huginn answers it from its stored forks, but with no question there is nothing to key a record under). Each gets a `refusal` and is dropped. A malformed body is therefore never stored, and neither is an unknown fork. Every refusal from step 4 on is stored. **`unavailable`** (503) is not a refusal and not in the refusal set: Bifrost answers it when the second door or Huginn did not answer after the auth steps and the body shape passed (a bad body is still refused first). **It says the outcome is NOT CONFIRMED, never that nothing was stored**: a timeout after the request reached Huginn may have stored it. It carries no answer id. The desk retries the same request with the same `request_id` and learns the outcome: the stored one if Huginn had it, a fresh one if not. It covers **calls 1 and 2 as well as the answer POST** when Huginn cannot be reached (Transport; a read has no outcome to confirm, so a retry is plain). `retryAfterSeconds` is nil when Bifrost has no hint.
|
`received_at` is Huginn's admission time; "first" means first by admission order (the `n<seq>` ordinal), never by `answered_at`. `claim_in_future` is true when `answered_at` is more than five minutes after `received_at`: accepted but flagged. It is a function of the two stored values, so the writer compares against the `received_at` it stores, at the precision it stores it (a flag decided against an unrounded clock and a stored second-truncated `received_at` disagrees with the document it sits in). **Not stored, and no answer id:** a body refused at step 1 or 2 (Bifrost; it never reaches Huginn), and a body naming an unknown fork (step 3; Huginn answers it from its stored forks, but with no question there is nothing to key a record under). Each gets a `refusal` and is dropped. A malformed body is therefore never stored, and neither is an unknown fork. Every refusal from step 4 on is stored. **`unavailable`** (503) is not a refusal and not in the refusal set: Bifrost answers it when the second door or Huginn did not answer after the auth steps and the body shape passed (a bad body is still refused first). **It says the outcome is NOT CONFIRMED, never that nothing was stored**: a timeout after the request reached Huginn may have stored it. It carries no answer id. The desk retries the same request with the same `request_id` and learns the outcome: the stored one if Huginn had it, a fresh one if not. It covers **calls 1 and 2 as well as the answer POST** when Huginn cannot be reached (Transport; a read has no outcome to confirm, so a retry is plain). `retryAfterSeconds` is nil when Bifrost has no hint.
|
||||||
|
|
||||||
**The mind's record and this document.** Huginn keeps the answer as its own record kind (`epiphany.pipeline.desk_answer.v2`, key `<campaign>:desk_answer:<question-label>.n<seq>`, the outcome as one externally tagged value, the fork as `fork`). The second door projects it to this document and nothing else crosses: `answer_id` is the key's `n<seq>` part, `fork_id` is the stored `fork`, the tagged outcome flattens into `outcome`, `of`, `retracts` and `code`, and the six refusal codes are spelt as in the schema (snake_case; the record's enum spells them in the mind's own PascalCase). On the wire an answer is named by its `n<seq>` alone, always inside the fork whose `fork_id` the document carries; a runtime that holds the full record key converts at the door, in one place. `body_sha256` and any other digest Huginn keeps to decide "same body" stay in the record: Huginn alone decides a replay or a `request_id_conflict` and Bifrost never compares bodies, so no digest is a field of this document. A digest is a valid way to decide it only if it covers `fork_id`, `options_hash`, `answer.choice`, `answer.note`, `retracted`, `retracts` and `answered_at` (every field of the request but `request_id`), is computed by Huginn and never read from a client.
|
**The record.** `record` is `{schemaVersion, fork_id, options_hash, counted_answer_id, answerer_count, answers[]}`. `answerer_count` is an integer of at least 0: the number of distinct answerer ids over every stored answer of the fork, refusals included (a refusal stores the stamped answerer too), and it is the same number in both views (vectors `answerer_count_missing`, `_negative`, `_null`). `answers[].answerer` is a string or nil (nil only in the non-decider view).
|
||||||
|
|
||||||
|
**The mind's record and this document.** Huginn keeps the answer as its own record kind (`epiphany.pipeline.desk_answer.v2`, key `<campaign>:desk_answer:<question-label>.n<seq>`, the outcome as one externally tagged value, the fork as `fork`). The second door projects it to this document and nothing else crosses: `answer_id` is the key's `n<seq>` part, `fork_id` is the stored `fork`, the tagged outcome flattens into `outcome`, `of`, `retracts` and `code`, and the six refusal codes are spelt as in the schema (snake_case; the record's enum spells them in the mind's own PascalCase). On the wire an answer is named by its `n<seq>` alone, always inside the fork whose `fork_id` the document carries; a runtime that holds the full record key converts at the door, in one place. Huginn's stored `desk_answer` keeps a non-nil `answerer`; nil exists only in a record projected to a non-decider. `body_sha256` and any other digest Huginn keeps to decide "same body" stay in the record: Huginn alone decides a replay or a `request_id_conflict` and Bifrost never compares bodies, so no digest is a field of this document. A digest is a valid way to decide it only if it covers `fork_id`, `options_hash`, `answer.choice`, `answer.note`, `retracted`, `retracts` and `answered_at` (every field of the request but `request_id`), is computed by Huginn and never read from a client.
|
||||||
|
|
||||||
**Huginn's digest, pinned in the vectors (`bodyDigests`; informative, not a wire document).** SHA-256 of one compact JSON object as `serde_json::to_string` writes it (UTF-8, no spaces), lowercase hex, keys in this order, `null` for a missing value, strings verbatim: `fork`, `options_hash`, `answered_at`, `choice`, `note`, `retracts` (the **full stored key** `<campaign>:desk_answer:<label>.n<seq>`: the door converts the wire's `n<seq>` before the digest), then, **only for the three illegal forms** (R1: a retraction carrying an answer, a retraction naming no target, a plain answer carrying a target), `illegal_retraction: true` and `retracted: true|false`. `request_id` is not in it. A plain answer and a legal retraction carry neither trailing key, so their digests are the ones Huginn main pins (`8a4d5b2e`, `9f807538`) and do not move. `9f807538` is pinned for a target no door produces (`Q1.1`); the golden for the real legal retraction is the door's form, the wire's `n1` converted to `Q1.n1` (`84576840`, vector `legal_retraction_door_form`, also main-valid). Source: Eureka's `desk-body-digest-order-20261110.md`, computed with `sha256sum`; the generator, the C# suite and the TypeScript suite recompute every digest from the input fields. **Status:** the three `retracted` goldens (`935cab1a`, `82b24ccc`, `34ef86ed`, and `79eee623` for form (i) with a null target) are the form of `cut-desk-digest-retracted.r2`, merged as Huginn #30. Eureka reports that these four and the door-form legal retraction (`84576840`) reproduce byte for byte on Huginn main `f0ca504` (the #30 fix); that run is hers, reported to us, and was not rerun here. Before #30, Huginn main (`7ea11b4`) wrote `illegal_retraction` alone, so forms (i) and (iii) collided there (`69074cc6`, pinned as `illegal_i_and_iii_alike_on_main`). The wire contract does not wait on it: the conflict scenarios are pinned in `scenarios`.
|
**Huginn's digest, pinned in the vectors (`bodyDigests`; informative, not a wire document).** SHA-256 of one compact JSON object as `serde_json::to_string` writes it (UTF-8, no spaces), lowercase hex, keys in this order, `null` for a missing value, strings verbatim: `fork`, `options_hash`, `answered_at`, `choice`, `note`, `retracts` (the **full stored key** `<campaign>:desk_answer:<label>.n<seq>`: the door converts the wire's `n<seq>` before the digest), then, **only for the three illegal forms** (R1: a retraction carrying an answer, a retraction naming no target, a plain answer carrying a target), `illegal_retraction: true` and `retracted: true|false`. `request_id` is not in it. A plain answer and a legal retraction carry neither trailing key, so their digests are the ones Huginn main pins (`8a4d5b2e`, `9f807538`) and do not move. `9f807538` is pinned for a target no door produces (`Q1.1`); the golden for the real legal retraction is the door's form, the wire's `n1` converted to `Q1.n1` (`84576840`, vector `legal_retraction_door_form`, also main-valid). Source: Eureka's `desk-body-digest-order-20261110.md`, computed with `sha256sum`; the generator, the C# suite and the TypeScript suite recompute every digest from the input fields. **Status:** the three `retracted` goldens (`935cab1a`, `82b24ccc`, `34ef86ed`, and `79eee623` for form (i) with a null target) are the form of `cut-desk-digest-retracted.r2`, merged as Huginn #30. Eureka reports that these four and the door-form legal retraction (`84576840`) reproduce byte for byte on Huginn main `f0ca504` (the #30 fix); that run is hers, reported to us, and was not rerun here. Before #30, Huginn main (`7ea11b4`) wrote `illegal_retraction` alone, so forms (i) and (iii) collided there (`69074cc6`, pinned as `illegal_i_and_iii_alike_on_main`). The wire contract does not wait on it: the conflict scenarios are pinned in `scenarios`.
|
||||||
|
|
||||||
|
|
@ -113,8 +117,8 @@ Source: the desk's client needs (Ratatoskr PR #24 section 12, `ratatoskr-desk-sp
|
||||||
- **M1.** `{mind}` is looked up in a **fixed configured list** of Mind ids, each mapped in configuration to that Mind's socket on the peer-checked door. The segment is percent-decoded once and compared, ordinally, with the list's ids: no case folding, trimming or Unicode normalisation, and a decoding failure is a miss. It is **never joined into a path, a socket name or a query**, so a value off the list opens nothing (`../x`, `a/b`, `a%2fb`, `eureka%00`, a 4 KB string, `GAMECULT-CHAT` for `gamecult-chat`, a trailing space, bytes that are not UTF-8: all unknown, 404, no socket opened).
|
- **M1.** `{mind}` is looked up in a **fixed configured list** of Mind ids, each mapped in configuration to that Mind's socket on the peer-checked door. The segment is percent-decoded once and compared, ordinally, with the list's ids: no case folding, trimming or Unicode normalisation, and a decoding failure is a miss. It is **never joined into a path, a socket name or a query**, so a value off the list opens nothing (`../x`, `a/b`, `a%2fb`, `eureka%00`, a 4 KB string, `GAMECULT-CHAT` for `gamecult-chat`, a trailing space, bytes that are not UTF-8: all unknown, 404, no socket opened).
|
||||||
- **M2.** An unknown Mind gets **the same refusal as an unknown fork**: `fork_unknown`, 404, the same bytes (the document's `error` names neither the Mind nor the fork). The list of Minds is not probeable. Auth is judged first, so a caller without a valid token, scope or Active membership learns nothing about which Minds exist.
|
- **M2.** An unknown Mind gets **the same refusal as an unknown fork**: `fork_unknown`, 404, the same bytes (the document's `error` names neither the Mind nor the fork). The list of Minds is not probeable. Auth is judged first, so a caller without a valid token, scope or Active membership learns nothing about which Minds exist.
|
||||||
- **M3.** On the POST the route's Mind must hold `fork_id`: step 3 is judged against **that Mind's** stored forks, so a `fork_id` the route's Mind does not hold is `fork_unknown` and nothing is stored, in either Mind. A fork cannot be answered through another Mind's route. A body that names `mind` is `body_unknown_field`: the Mind is the route's.
|
- **M3.** On the POST the route's Mind must hold `fork_id`: step 3 is judged against **that Mind's** stored forks, so a `fork_id` the route's Mind does not hold is `fork_unknown` and nothing is stored, in either Mind. A fork cannot be answered through another Mind's route. A body that names `mind` is `body_unknown_field`: the Mind is the route's.
|
||||||
- **M4.** Scopes are **not per Mind**: `desk.read` and `desk.answer` open the routes of every configured Mind; every reader sees every fork's card and outcome, and which forks a caller may answer is the decider list.
|
- **M4.** Scopes are **not per Mind**: `desk.read` and `desk.answer` open the routes of every configured Mind; every reader sees every fork's card and outcome (a non-decider sees no account id, "Reader views"), and which forks a caller may answer is the decider list.
|
||||||
- **M5. The decider list is sent per call.** Bifrost holds the admin-set list and sends it with **every answer, read and list request** to Huginn, as a typed field on the same peer-checked door; Huginn keeps no copy of it. Because it travels with read and list too, a projected card's `deciders` is the list of that call, and step 5 judges an answer against the list the same call carried. It comes only from Bifrost's admin-set state: never from a Mind and never from any request body a client controls: no HTTP field, header or query parameter on any desk route sets or extends it, and a body naming `deciders` is `body_unknown_field` (scenario `M5_...`). A request carrying it from a Mind's own door or session is refused. **This is Bifrost and Huginn work and no vector here can test it** (the typed field and the door's wire are pending, open choice 3); the rule is stated so that the cut that builds the request has it. (Heimdall's ruling, #minds 2026-11-10 19:26Z; it replaces the earlier reading of a list Huginn receives once and holds.) **Envelope names:** the desk's envelope on that door uses the names this page uses, `mind`, `authority`, `subject` and `deciders`, with no conversion at the door (Eureka, #minds 2026-11-10 19:58Z); `deciders` is a per-call list of short ids at the envelope's top level beside `subject`, never inside the request body, and Huginn keeps no copy.
|
- **M5. The decider list is sent per call.** Bifrost holds the admin-set list and sends it with **every answer, read and list request** to Huginn, as a typed field on the same peer-checked door; Huginn keeps no copy of it. Because it travels with read and list too, a projected card's `deciders` is the list of that call for a decider and empty for a reader who is not one, with `decider_count` travelling with both views, and step 5 judges an answer against the list the same call carried. It comes only from Bifrost's admin-set state: never from a Mind and never from any request body a client controls: no HTTP field, header or query parameter on any desk route sets or extends it, and a body naming `deciders` is `body_unknown_field` (scenario `M5_...`). A request carrying it from a Mind's own door or session is refused. **This is Bifrost and Huginn work and no vector here can test it** (the typed field and the door's wire are pending, open choice 3); the rule is stated so that the cut that builds the request has it. (Heimdall's ruling, #minds 2026-11-10 19:26Z; it replaces the earlier reading of a list Huginn receives once and holds.) **Envelope names:** the desk's envelope on that door uses the names this page uses, `mind`, `authority`, `subject` and `deciders`, with no conversion at the door (Eureka, #minds 2026-11-10 19:58Z); `deciders` is a per-call list of short ids at the envelope's top level beside `subject`, never inside the request body, and Huginn keeps no copy.
|
||||||
|
|
||||||
Calls 1 and 2 are enough for a read-only desk. Every response is the pinned MessagePack document or its JSON form with the same keys (open choice 1 of `vote-api.md`: the media type is unsettled).
|
Calls 1 and 2 are enough for a read-only desk. Every response is the pinned MessagePack document or its JSON form with the same keys (open choice 1 of `vote-api.md`: the media type is unsettled).
|
||||||
|
|
||||||
|
|
@ -122,7 +126,7 @@ Calls 1 and 2 are enough for a read-only desk. Every response is the pinned Mess
|
||||||
|
|
||||||
**List.** `list_request`: `{mind, status, cursor, limit}`. `mind` is the route's `{mind}`, a Mind id from the configured list. `status` is `open`, `recorded` or nil for both; `recorded` means a closed card. Cards come **newest first by `order`**. `order` is the card's unique rank per Mind: an integer from 0 that never changes once the card exists, later questions only appending (Huginn derives it from the Mind's questions sorted by admission ordinal, then key). A Mind never holds two cards of one `order`; a list or card set that does is refused, and the refusal names both fork ids. 0 is legal, but no cursor names it (`c0` fails the cursor pattern): a page ending at order 1 returns `c1`, and the next page returns the order-0 card alone with `next_cursor` nil. `limit` is 0 to 50; **`limit` 0 returns no entries, only the counts and the stamp**, which is what "N open" on the Mind's header needs. `cursor` is nil or `c<N>`: the cards whose `order` is below N; a client treats it as opaque and sends back the `next_cursor` it was given. `list`: `{mind, stamp, open_count, recorded_count, forks[entry], next_cursor}`. The counts cover every card the viewer may see, whatever `status` and `cursor` asked for. `stamp` is `s<N>`, N the receiver's last-seen Huginn admission ordinal for the Mind's desk documents: the same stamp means the same state, so a desk polls `limit` 0 and fetches bodies only when the stamp moves. `next_cursor` is nil on the last page. **A page may hold fewer than `limit` entries** when the next entry would push the encoded response past the receiver's response budget (Huginn: 65536 bytes); it then holds at least one, `next_cursor` is `c<order of the last entry returned>` so the next page resumes below it, and the counts and the stamp are those of the full list, unchanged. **`fork_too_large`** (HTTP 422) refuses a page whose **first** entry alone is past the budget: 422 because the request is well formed and authorised but names a resource the list cannot carry, and 413 is for a request body that is too large, which this is not; retrying returns the same refusal. Its `error` names the fork id and its `order`, for example `fork_too_large: fork <fork_id> (order <n>) alone is past the response budget`. **A refusal's `error` text is informational and not pinned**: the contract pins the code, the HTTP status, that the text names the fork id (and the order on a list), and the `c<order>` continuation on a list; the vectors' wording is a sample, and the judges check those four things, not the bytes of the text. The client continues the list with cursor `c<n>`, the cards strictly below that order, which skips this fork; it is then readable only by id (the read route returns the whole entry, and refuses with the same code only when that entry alone is past the cap, see Read); `c<n>` is valid only when n is at least 1, so for order 0 there is nothing below and the client has reached the end. `limit` 0 returns no entries and never refuses. (Ruled by Persona, 2026-11-11.) An `entry` is `{card, record}`: the card and its `record` (the answers with their outcomes and the derived `counted_answer_id`).
|
**List.** `list_request`: `{mind, status, cursor, limit}`. `mind` is the route's `{mind}`, a Mind id from the configured list. `status` is `open`, `recorded` or nil for both; `recorded` means a closed card. Cards come **newest first by `order`**. `order` is the card's unique rank per Mind: an integer from 0 that never changes once the card exists, later questions only appending (Huginn derives it from the Mind's questions sorted by admission ordinal, then key). A Mind never holds two cards of one `order`; a list or card set that does is refused, and the refusal names both fork ids. 0 is legal, but no cursor names it (`c0` fails the cursor pattern): a page ending at order 1 returns `c1`, and the next page returns the order-0 card alone with `next_cursor` nil. `limit` is 0 to 50; **`limit` 0 returns no entries, only the counts and the stamp**, which is what "N open" on the Mind's header needs. `cursor` is nil or `c<N>`: the cards whose `order` is below N; a client treats it as opaque and sends back the `next_cursor` it was given. `list`: `{mind, stamp, open_count, recorded_count, forks[entry], next_cursor}`. The counts cover every card the viewer may see, whatever `status` and `cursor` asked for. `stamp` is `s<N>`, N the receiver's last-seen Huginn admission ordinal for the Mind's desk documents: the same stamp means the same state, so a desk polls `limit` 0 and fetches bodies only when the stamp moves. `next_cursor` is nil on the last page. **A page may hold fewer than `limit` entries** when the next entry would push the encoded response past the receiver's response budget (Huginn: 65536 bytes); it then holds at least one, `next_cursor` is `c<order of the last entry returned>` so the next page resumes below it, and the counts and the stamp are those of the full list, unchanged. **`fork_too_large`** (HTTP 422) refuses a page whose **first** entry alone is past the budget: 422 because the request is well formed and authorised but names a resource the list cannot carry, and 413 is for a request body that is too large, which this is not; retrying returns the same refusal. Its `error` names the fork id and its `order`, for example `fork_too_large: fork <fork_id> (order <n>) alone is past the response budget`. **A refusal's `error` text is informational and not pinned**: the contract pins the code, the HTTP status, that the text names the fork id (and the order on a list), and the `c<order>` continuation on a list; the vectors' wording is a sample, and the judges check those four things, not the bytes of the text. The client continues the list with cursor `c<n>`, the cards strictly below that order, which skips this fork; it is then readable only by id (the read route returns the whole entry, and refuses with the same code only when that entry alone is past the cap, see Read); `c<n>` is valid only when n is at least 1, so for order 0 there is nothing below and the client has reached the end. `limit` 0 returns no entries and never refuses. (Ruled by Persona, 2026-11-11.) An `entry` is `{card, record}`: the card and its `record` (the answers with their outcomes and the derived `counted_answer_id`).
|
||||||
|
|
||||||
**Read.** `read_request` is `{mind, fork_id}`; the response is the `entry` for one fork of that Mind. Its `card.options_hash` is **computed by the server** from the five shown members as in "Options hash" and the client recomputes it from what it rendered; a card whose hash is not the hash of its members is no card (negative vector `card_hash_wrong`). The record's `fork_id` and `options_hash` are the card's, and `counted_answer_id` is derived, not copied (negative vectors for each). Every `desk.read` holder sees every fork's card and outcome in a configured Mind; only a decider's notes depend on the caller, going to callers in the admin-set `deciders` list, which Bifrost enforces at the door (ruling `desk-read-visibility`, option (c)); `fork_unknown` is only for a fork id that does not exist under the route's Mind (an unknown Mind answers the same, M2), so no fork is ever hidden by it, and a caller without `desk.read` is refused by the auth order above (`insufficient_scope`) before any lookup. **Bifrost applies visibility at the door; Huginn returns the whole record to the door.** Every `desk.read` holder sees cards and outcomes; notes (a decider's own words) only go to callers in the admin-set `deciders` list. The read and list shapes are unchanged and carry no audience field (`eureka-substrate:question:desk-read-visibility`, Heimdall's option (c)). **A read is refused `fork_too_large` (422) when the whole entry is past the reply cap** (Huginn's daemon caps every reply at 65536 bytes; the cap, not the page budget, is what a read meets). Its `error` names the fork id and has no order and no cursor to carry, because there is nothing to continue with (sample: `fork_too_large: fork <fork_id> alone is past the response budget`) (vector `transport.readTooLarge`; the cap is in no vector). **The answer route meets the same cap:** when an answer is `stale_fork`, Bifrost re-reads the current card to attach it, and if that read is past the cap the answer is refused `fork_too_large` (422) with the read form of the text, the same document as a read. Huginn has already stored the answer as `Refused(stale_fork)`, as for any stale answer, and it is never counted; the client is told `fork_too_large` instead of receiving the current card, and a replay of the same `request_id` gets the same 422 while the fork stays oversize. It is never `unavailable` (vector `transport.answerTooLarge`). A stripped read serves a decider's `note` as `note: null`, never omitted (map-mode: every key is written); `desk_answer.note` is already nullable in the schema, so a served record validates as it stands.
|
**Read.** `read_request` is `{mind, fork_id}`; the response is the `entry` for one fork of that Mind. Its `card.options_hash` is **computed by the server** from the five shown members as in "Options hash" and the client recomputes it from what it rendered; a card whose hash is not the hash of its members is no card (negative vector `card_hash_wrong`). The record's `fork_id` and `options_hash` are the card's, and `counted_answer_id` is derived, not copied (negative vectors for each). Every `desk.read` holder sees every fork's card and outcome in a configured Mind; only a decider's notes depend on the caller, going to callers in the admin-set `deciders` list, which Bifrost enforces at the door (ruling `desk-read-visibility`, option (c)); `fork_unknown` is only for a fork id that does not exist under the route's Mind (an unknown Mind answers the same, M2), so no fork is ever hidden by it, and a caller without `desk.read` is refused by the auth order above (`insufficient_scope`) before any lookup. **Huginn withholds account ids from a reader who is not a decider and returns counts ("Reader views"); Bifrost still strips notes.** Every `desk.read` holder sees cards and outcomes; notes (a decider's own words) only go to callers in the admin-set `deciders` list. The read and list shapes are unchanged and carry no audience field (`eureka-substrate:question:desk-read-visibility`, Heimdall's option (c)). **A read is refused `fork_too_large` (422) when the whole entry is past the reply cap** (Huginn's daemon caps every reply at 65536 bytes; the cap, not the page budget, is what a read meets). Its `error` names the fork id and has no order and no cursor to carry, because there is nothing to continue with (sample: `fork_too_large: fork <fork_id> alone is past the response budget`) (vector `transport.readTooLarge`; the cap is in no vector). **The answer route meets the same cap:** when an answer is `stale_fork`, Bifrost re-reads the current card to attach it, and if that read is past the cap the answer is refused `fork_too_large` (422) with the read form of the text, the same document as a read. Huginn has already stored the answer as `Refused(stale_fork)`, as for any stale answer, and it is never counted; the client is told `fork_too_large` instead of receiving the current card, and a replay of the same `request_id` gets the same 422 while the fork stays oversize. It is never `unavailable` (vector `transport.answerTooLarge`). A stripped read serves a decider's `note` as `note: null`, never omitted (map-mode: every key is written); `desk_answer.note` is already nullable in the schema, so a served record validates as it stands.
|
||||||
|
|
||||||
**Answer.** The body is the `answer_request` and nothing else a client may set: `fork_id`, `options_hash`, `answer`, `answered_at`, `request_id` (plus `retracted` and `retracts`, written `false` and nil unless the call retracts). The response is the stored outcome: `Counted`, `Revision`, `Retraction`, or `Refused` with its code, and the **`counted_answer_id`** the fork has once this call is applied (`n<seq>` or nil; derived as in "The counted answer is derived", the same value the `record` projection carries, so the desk needs no second read). A replay returns the stored outcome with the `counted_answer_id` current at the retry, not the one it had when first stored. **A `stale_fork` refusal carries the current card** in `answer_result.card`, so the desk can show the new fork without another read. Nothing is stored for a step 1 or 2 refusal, an unknown fork (or an unknown Mind, or a fork another Mind holds) or a `request_id_conflict`. `unavailable` promises nothing either way.
|
**Answer.** The body is the `answer_request` and nothing else a client may set: `fork_id`, `options_hash`, `answer`, `answered_at`, `request_id` (plus `retracted` and `retracts`, written `false` and nil unless the call retracts). The response is the stored outcome: `Counted`, `Revision`, `Retraction`, or `Refused` with its code, and the **`counted_answer_id`** the fork has once this call is applied (`n<seq>` or nil; derived as in "The counted answer is derived", the same value the `record` projection carries, so the desk needs no second read). A replay returns the stored outcome with the `counted_answer_id` current at the retry, not the one it had when first stored. **A `stale_fork` refusal carries the current card** in `answer_result.card`, so the desk can show the new fork without another read. Nothing is stored for a step 1 or 2 refusal, an unknown fork (or an unknown Mind, or a fork another Mind holds) or a `request_id_conflict`. `unavailable` promises nothing either way.
|
||||||
|
|
||||||
|
|
@ -149,7 +153,9 @@ The old desk (`index.html`) read `questions/<id>` and wrote `answers/<id>` in a
|
||||||
| `status` open or closed, `ruling` | derived from the resolution (status is never written): any resolution of the question closes it; the ruling is the one that answered it | `status`, `ruling`, both a projection (`ruling` is nil on an open card and on a card closed with no ruling) |
|
| `status` open or closed, `ruling` | derived from the resolution (status is never written): any resolution of the question closes it; the ruling is the one that answered it | `status`, `ruling`, both a projection (`ruling` is nil on an open card and on a card closed with no ruling) |
|
||||||
| `order` | none (unique rank per Mind, from 0) | `order` |
|
| `order` | none (unique rank per Mind, from 0) | `order` |
|
||||||
| none | `raised_in`, `asked_on` | `raised_in`; `asked_on` is not carried |
|
| none | `raised_in`, `asked_on` | `raised_in`; `asked_on` is not carried |
|
||||||
|
| none | the question's admission time (no timestamp in the old format; `asked_on` is a date and stays uncarried) | `raised_at`, an instant or nil, not hashed |
|
||||||
| none | none | `deciders` (outside the Mind), `ruled_from`, `options_hash` |
|
| none | none | `deciders` (outside the Mind), `ruled_from`, `options_hash` |
|
||||||
|
| none | none | `decider_count`, `answerer_count` (counts that stand in for withheld ids) |
|
||||||
| answer `{choice: key or null, note, question, updatedAt}`, one mutable document per question | none | `answer_request` then an append-only `desk_answer` |
|
| answer `{choice: key or null, note, question, updatedAt}`, one mutable document per question | none | `answer_request` then an append-only `desk_answer` |
|
||||||
| `note`: her words verbatim | | `answer.note`, verbatim |
|
| `note`: her words verbatim | | `answer.note`, verbatim |
|
||||||
| "Clear pick": `choice` back to null so the note stands alone | | an `answer` with `choice` nil and `note` set, as the first answer or as a `Revision` |
|
| "Clear pick": `choice` back to null so the note stands alone | | an `answer` with `choice` nil and `note` set, as the first answer or as a `Revision` |
|
||||||
|
|
@ -178,6 +184,8 @@ Option B, **Bifrost receives the answer and Huginn stores it** (the design in th
|
||||||
- `request_id` (on the request and the stored `desk_answer`) and `request_id_conflict` are this page's: his file has no idempotency key, and a retry after `unavailable` needs one. Eureka's `desk_answer` kind and Huginn's request type must carry it.
|
- `request_id` (on the request and the stored `desk_answer`) and `request_id_conflict` are this page's: his file has no idempotency key, and a retry after `unavailable` needs one. Eureka's `desk_answer` kind and Huginn's request type must carry it.
|
||||||
- Transport: the routes, the `{mind}` segment, the scopes, the audience and the auth order are his rulings (4.1 to 4.3a); the documents (list, read, `unavailable`) are this page's. The note limit is 4000 UTF-8 bytes here, as his 1.1 leaves the unit unsaid.
|
- Transport: the routes, the `{mind}` segment, the scopes, the audience and the auth order are his rulings (4.1 to 4.3a); the documents (list, read, `unavailable`) are this page's. The note limit is 4000 UTF-8 bytes here, as his 1.1 leaves the unit unsaid.
|
||||||
- The recommendation is never nil. The binding file's earlier vectors D and E used a nil recommendation; its updated D and E use `"a"`, as here.
|
- The recommendation is never nil. The binding file's earlier vectors D and E used a nil recommendation; its updated D and E use `"a"`, as here.
|
||||||
|
- `raised_at` (2026-11-11) is the desk-feed addition (Bifrost PR 67): a required key, an instant or nil, outside `options_hash` (the five hashed members and every digest are unchanged; vectors `raised_at_nil`, `raised_at_epoch_integer`, `raised_at_date_only`, `raised_at_missing`). The card is spec-authored and no runtime deploys it, so the schema id stays `gamecult.fork.card.v0`.
|
||||||
|
- Reader views (2026-11-11) are the desk-feed addition behind Huginn PR 56 and `eureka-substrate:ruling:desk-read-visibility-withheld-ids`: the card gains `decider_count` and `deciders` may be empty; the record gains `answerer_count` and `answerer` may be nil, the counts beside the collections they summarise (not on the list envelope, which neither the read entry nor the `stale_fork` card travels in). `options_hash` and every digest are unchanged. The schema ids stay `gamecult.fork.card.v0` and `gamecult.fork.record.v0` for the same reason as `raised_at`.
|
||||||
- Hash vectors: A, B, C, D, E, F and G of the binding file are reproduced digest for digest by the Python generator, the C# suite and the TypeScript suite (the generator prints `same` for each; none differs).
|
- Hash vectors: A, B, C, D, E, F and G of the binding file are reproduced digest for digest by the Python generator, the C# suite and the TypeScript suite (the generator prints `same` for each; none differs).
|
||||||
|
|
||||||
## Open choices
|
## Open choices
|
||||||
|
|
@ -186,7 +194,7 @@ Option B, **Bifrost receives the answer and Huginn stores it** (the design in th
|
||||||
2. NFC across runtimes relies on each runtime's normaliser; a pinned table, as `write-receipt.md` does for letters, would remove the reliance. The risk is only for code points assigned after a runtime's Unicode version.
|
2. NFC across runtimes relies on each runtime's normaliser; a pinned table, as `write-receipt.md` does for letters, would remove the reliance. The risk is only for code points assigned after a runtime's Unicode version.
|
||||||
3. How Huginn learns the verified subject: the second door's authenticated context (the peer credential names Bifrost the service, not the member), so the subject must travel beside the request. This contract has no envelope for it.
|
3. How Huginn learns the verified subject: the second door's authenticated context (the peer credential names Bifrost the service, not the member), so the subject must travel beside the request. This contract has no envelope for it.
|
||||||
4. Whether `fork_unknown` and shape refusals should leave an audit record somewhere other than a `desk_answer`.
|
4. Whether `fork_unknown` and shape refusals should leave an audit record somewhere other than a `desk_answer`.
|
||||||
5. Whether `leans` and `flag`, which the booth shows, should join the hash. The binding file's five members do not include them; a changed lean would not make an answer stale.
|
5. Whether `leans` and `flag`, which the booth shows, should join the hash. The binding file's five members do not include them; a changed lean would not make an answer stale. `raised_at` is immutable, so it was kept out of the hash for the same reason it need not be in: it can never make a shown card stale.
|
||||||
6. What the person sees of a retracted counted answer when the next answer counts: this contract only says the count moves.
|
6. What the person sees of a retracted counted answer when the next answer counts: this contract only says the count moves.
|
||||||
7. `received_at` is not in the binding file's stored fields; it is kept for `claim_in_future` and could be dropped if the `n<seq>` answer ordinal alone decided answer order.
|
7. `received_at` is not in the binding file's stored fields; it is kept for `claim_in_future` and could be dropped if the `n<seq>` answer ordinal alone decided answer order.
|
||||||
8. The stamped `authority` field is an attribution, not an authority (above); renaming it `attribution` would say so, at the price of a new key the door must also refuse in a body.
|
8. The stamped `authority` field is an attribution, not an authority (above); renaming it `attribution` would say so, at the price of a new key the door must also refuse in a body.
|
||||||
|
|
@ -195,3 +203,4 @@ Option B, **Bifrost receives the answer and Huginn stores it** (the design in th
|
||||||
11. While a known Mind's door is down its routes answer `unavailable` (503) and an unknown Mind answers `fork_unknown` (404), so the two are distinguishable for that period. M2 makes them identical only when every door is up; making them identical when one is down means answering `unavailable` for any `{mind}` not on the list, which tells a client nothing about which Minds exist but also hides a typo. Heimdall's to rule; the vectors do not pin it.
|
11. While a known Mind's door is down its routes answer `unavailable` (503) and an unknown Mind answers `fork_unknown` (404), so the two are distinguishable for that period. M2 makes them identical only when every door is up; making them identical when one is down means answering `unavailable` for any `{mind}` not on the list, which tells a client nothing about which Minds exist but also hides a typo. Heimdall's to rule; the vectors do not pin it.
|
||||||
12. A fork's answers are unbounded in number, so a fork's entry can grow past the reply cap and become unreadable (`fork_too_large` on read, and on list when it is the first entry of a page). Whether the contract should cap the answers per fork is for Persona and Heimdall to rule; nothing here decides it.
|
12. A fork's answers are unbounded in number, so a fork's entry can grow past the reply cap and become unreadable (`fork_too_large` on read, and on list when it is the first entry of a page). Whether the contract should cap the answers per fork is for Persona and Heimdall to rule; nothing here decides it.
|
||||||
13. The read form of `fork_too_large` is a target, not current behaviour: Huginn's door answers an over-cap read as `Refused(ResponseTooLarge{bytes, limit})`, which carries no fork id and is not `fork_too_large`. Bifrost maps it to `fork_too_large` (422) using the route's `fork_id`; until it does, a read past the cap reaches the client as 503 `unavailable`. The same holds for the answer route: a `stale_fork` answer whose card re-read is past the cap is a Bifrost target too (`fork_too_large` 422 over a stored `Refused(stale_fork)`), and until it is built the client sees 503 `unavailable`.
|
13. The read form of `fork_too_large` is a target, not current behaviour: Huginn's door answers an over-cap read as `Refused(ResponseTooLarge{bytes, limit})`, which carries no fork id and is not `fork_too_large`. Bifrost maps it to `fork_too_large` (422) using the route's `fork_id`; until it does, a read past the cap reaches the client as 503 `unavailable`. The same holds for the answer route: a `stale_fork` answer whose card re-read is past the cap is a Bifrost target too (`fork_too_large` 422 over a stored `Refused(stale_fork)`), and until it is built the client sees 503 `unavailable`.
|
||||||
|
14. What a non-decider can still infer: the card's `status` and `order`, the `authority` role on each answer (a role, not an id, and the ruling leaves it whole), and `answerer_count`. Whether a role plus a count is too much to show is Eureka's ruling to revisit.
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,9 @@
|
||||||
"leans",
|
"leans",
|
||||||
"flag",
|
"flag",
|
||||||
"raised_in",
|
"raised_in",
|
||||||
|
"raised_at",
|
||||||
"deciders",
|
"deciders",
|
||||||
|
"decider_count",
|
||||||
"status",
|
"status",
|
||||||
"ruling",
|
"ruling",
|
||||||
"ruled_from",
|
"ruled_from",
|
||||||
|
|
@ -143,15 +145,30 @@
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"raised_at": {
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/$defs/instant"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"deciders": {
|
"deciders": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"minItems": 1,
|
"minItems": 0,
|
||||||
"items": {
|
"items": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"minLength": 1
|
"minLength": 1
|
||||||
},
|
},
|
||||||
"maxItems": 64
|
"maxItems": 64
|
||||||
},
|
},
|
||||||
|
"decider_count": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1,
|
||||||
|
"maximum": 64
|
||||||
|
},
|
||||||
"status": {
|
"status": {
|
||||||
"enum": [
|
"enum": [
|
||||||
"open",
|
"open",
|
||||||
|
|
@ -373,7 +390,14 @@
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"answerer": {
|
"answerer": {
|
||||||
"$ref": "#/$defs/id"
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/$defs/id"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"authority": {
|
"authority": {
|
||||||
"$ref": "#/$defs/id"
|
"$ref": "#/$defs/id"
|
||||||
|
|
@ -459,7 +483,14 @@
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"answerer": {
|
"answerer": {
|
||||||
"$ref": "#/$defs/id"
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/$defs/id"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"authority": {
|
"authority": {
|
||||||
"$ref": "#/$defs/id"
|
"$ref": "#/$defs/id"
|
||||||
|
|
@ -531,7 +562,14 @@
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"answerer": {
|
"answerer": {
|
||||||
"$ref": "#/$defs/id"
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/$defs/id"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"authority": {
|
"authority": {
|
||||||
"$ref": "#/$defs/id"
|
"$ref": "#/$defs/id"
|
||||||
|
|
@ -631,7 +669,14 @@
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"answerer": {
|
"answerer": {
|
||||||
"$ref": "#/$defs/id"
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/$defs/id"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"authority": {
|
"authority": {
|
||||||
"$ref": "#/$defs/id"
|
"$ref": "#/$defs/id"
|
||||||
|
|
@ -735,6 +780,7 @@
|
||||||
"fork_id",
|
"fork_id",
|
||||||
"options_hash",
|
"options_hash",
|
||||||
"counted_answer_id",
|
"counted_answer_id",
|
||||||
|
"answerer_count",
|
||||||
"answers"
|
"answers"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
|
@ -757,6 +803,10 @@
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"answerer_count": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 0
|
||||||
|
},
|
||||||
"answers": {
|
"answers": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
|
|
|
||||||
|
|
@ -701,6 +701,7 @@ checked by the operator in the editor.
|
||||||
- Run `MintElementIds` once over `GameData/Aetheria.cc`.
|
- Run `MintElementIds` once over `GameData/Aetheria.cc`.
|
||||||
- AetherDb migrations skip variants or use the variant API (Q1a).
|
- AetherDb migrations skip variants or use the variant API (Q1a).
|
||||||
- Re-author the families listed in section 6.
|
- Re-author the families listed in section 6.
|
||||||
|
- Hull families live in each ship's own `.cc`, not in `GameData/Aetheria.cc`, because hulls are mod-owned packages (Nibu's ruling about her data, 2026-11-11, #minds).
|
||||||
|
|
||||||
### C5. CultNet deltas (C# reference)
|
### C5. CultNet deltas (C# reference)
|
||||||
|
|
||||||
|
|
@ -1608,7 +1609,7 @@ The one door per runtime, and who walks through it:
|
||||||
- **Shared paths.** cultnet-ts reaches the header decision only through `readSingleFileStore` (AUTH:4, :321) and checks only record count, schema id and type. cultnet-ts therefore needs a cultcache-ts release that exports it, which is the release that ships with this change; the manifests move to that version then (c0-ships-with-c2a), not before.
|
- **Shared paths.** cultnet-ts reaches the header decision only through `readSingleFileStore` (AUTH:4, :321) and checks only record count, schema id and type. cultnet-ts therefore needs a cultcache-ts release that exports it, which is the release that ships with this change; the manifests move to that version then (c0-ships-with-c2a), not before.
|
||||||
- **Cut line.** `merge-r1` h3 (Rust) and h4 (TS, cultnet-ts); spec `cut-merge-r1.r3` authority map.
|
- **Cut line.** `merge-r1` h3 (Rust) and h4 (TS, cultnet-ts); spec `cut-merge-r1.r3` authority map.
|
||||||
- **Pinned by.** `tests/vectors/document-variants-c2a/readability/manifest.txt`, walked by `StoreReadabilityTests.OpenReadsExactlyTheFilesTheVectorsSayItReads` (C#), `CultMeshSingleFileReadabilityTests.AWriteReplacesAFileExactlyWhenItReads` (Mesh), `a_file_is_replaced_exactly_when_it_reads` (RS:5132), "SingleFileMessagePackBackingStore replaces a file exactly when it reads" (TS test :2321) and `test_single_file_replaces_a_file_exactly_when_it_reads` (Python). AUTH cases: `packages/cultnet-ts/test/idunn-runtime-authority.test.ts`. Declared divergences in the manifest: `legacy-envelopes.bin` (C# refuses; the others read and replace it) and `variant-slot-v3.bin` (C# reads; the others refuse).
|
- **Pinned by.** `tests/vectors/document-variants-c2a/readability/manifest.txt`, walked by `StoreReadabilityTests.OpenReadsExactlyTheFilesTheVectorsSayItReads` (C#), `CultMeshSingleFileReadabilityTests.AWriteReplacesAFileExactlyWhenItReads` (Mesh), `a_file_is_replaced_exactly_when_it_reads` (RS:5132), "SingleFileMessagePackBackingStore replaces a file exactly when it reads" (TS test :2321) and `test_single_file_replaces_a_file_exactly_when_it_reads` (Python). AUTH cases: `packages/cultnet-ts/test/idunn-runtime-authority.test.ts`. Declared divergences in the manifest: `legacy-envelopes.bin` (C# refuses; the others read and replace it) and `variant-slot-v3.bin` (C# reads; the others refuse).
|
||||||
- **Known gaps.** `variants:follow_up:c0-bad-canonical-in-manifest` (`current-catalog-bad-canonical.msgpack` is not in the manifest; TS reads it, Rust refuses). `variants:follow_up:legacy-vectors-all-runtimes` (the three `legacy-catalog-*` c0 vectors have a verdict only in C# tests; all four runtimes refuse them, probed). `variants:follow_up:range-check-in-ci` (`npm run test:pack:ts` builds every admitted tag and checks exports, but `.github/workflows/publish-packages.yml` does not run it).
|
- **Known gaps.** `variants:follow_up:range-check-in-ci` (`npm run test:pack:ts` builds every admitted tag and checks exports, but `.github/workflows/publish-packages.yml` does not run it).
|
||||||
|
|
||||||
### B. Existence and links (what "nothing at the path" means)
|
### B. Existence and links (what "nothing at the path" means)
|
||||||
|
|
||||||
|
|
@ -1639,7 +1640,7 @@ The one door per runtime, and who walks through it:
|
||||||
- **Shared paths.** Open, commit, Mesh read and write.
|
- **Shared paths.** Open, commit, Mesh read and write.
|
||||||
- **Cut line.** `mesh-single-file-reader` h1 (the C# clause). Rust, TS and Python were correct by construction of their reads.
|
- **Cut line.** `mesh-single-file-reader` h1 (the C# clause). Rust, TS and Python were correct by construction of their reads.
|
||||||
- **Pinned by.** C#: `ADirectoryAtTheStorePathIsAnIoErrorOnOpenAndOnCommitNotAnEmptyStore` (`StoreReadabilityTests.cs:236`). Mesh: `ADirectoryAtThePathIsAnIoErrorOnReadAndWriteAndIsLeftAlone` (:149). Rust: `a_write_over_a_directory_stops_at_the_read_and_leaves_no_staging_file` RS:5726.
|
- **Pinned by.** C#: `ADirectoryAtTheStorePathIsAnIoErrorOnOpenAndOnCommitNotAnEmptyStore` (`StoreReadabilityTests.cs:236`). Mesh: `ADirectoryAtThePathIsAnIoErrorOnReadAndWriteAndIsLeftAlone` (:149). Rust: `a_write_over_a_directory_stops_at_the_read_and_leaves_no_staging_file` RS:5726.
|
||||||
- **Known gaps.** TS and Python have no test for a directory at the store path. No follow-up is registered; the natural home is `write-set-vectors` with the other readability pins.
|
- **Known gaps.** None open. The `@directory` manifest row is walked by all four runtimes: every operation including delete is refused and leaves the directory untouched; Rust pins an I/O failure rather than a typed unreadable store.
|
||||||
|
|
||||||
### E. Lock
|
### E. Lock
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ Data for the per-consumer sweep in docs/document-variants-cut.md, C2. Every obje
|
||||||
|
|
||||||
Method: a static scan of `F:\Projects\<consumer>` C# files for types marked `[CultDocument(`, walked through keyed members, base classes, dictionary values and `[Union]` subtypes. Read-only; no consumer was edited. Worktree copies (`Aetheria-catalog`, `CultLib-*`, and so on) are excluded. Struct elements are values and need no id. Non-C# consumers carry the id as an ordinary member and are not listed. The scan is text-based: a document registered without a literal `[CultDocument(` attribute, or a type whose members lack an integer `[Key]`, is missed, so the sweep pass should confirm each repo by registering its types once.
|
Method: a static scan of `F:\Projects\<consumer>` C# files for types marked `[CultDocument(`, walked through keyed members, base classes, dictionary values and `[Union]` subtypes. Read-only; no consumer was edited. Worktree copies (`Aetheria-catalog`, `CultLib-*`, and so on) are excluded. Struct elements are values and need no id. Non-C# consumers carry the id as an ordinary member and are not listed. The scan is text-based: a document registered without a literal `[CultDocument(` attribute, or a type whose members lack an integer `[Key]`, is missed, so the sweep pass should confirm each repo by registering its types once.
|
||||||
|
|
||||||
|
A type that already has a readable string id keeps it and adds a trailing keyed string member marked `[CultElementId(nameof(ExistingId))]`; see the Element ids section of `src/GameCult.Caching/README.md`.
|
||||||
|
|
||||||
Lines: `Type file:line (via Owner.Member)`, paths relative to the consumer root.
|
Lines: `Type file:line (via Owner.Member)`, paths relative to the consumer root.
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -25,5 +25,13 @@ closure through UPM rather than NuGet restore.
|
||||||
Install the Unity package from the immutable release tag:
|
Install the Unity package from the immutable release tag:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v2.0.0
|
https://github.com/GameCult/CultLib.git?path=/unity/org.gamecult.cultlib#cultlib-unity-v3.0.0
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Rolling back across the store format
|
||||||
|
|
||||||
|
A consumer that upgrades and writes a store holding an element id leaves a
|
||||||
|
`cultcache.store.v3` single-file store (`v5` directory store). Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. There is
|
||||||
|
no general carry-back; restore from a copy taken before upgrading, and upgrade
|
||||||
|
readers before writers. Detail: the Store format and rollback section of
|
||||||
|
`src/GameCult.Caching/README.md`.
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ Cross-runtime parity has three layers:
|
||||||
Every store reader refuses, by name, a header or record slot it does not
|
Every store reader refuses, by name, a header or record slot it does not
|
||||||
understand (vectors: `tests/vectors/document-variants-c0`).
|
understand (vectors: `tests/vectors/document-variants-c0`).
|
||||||
Element ids: a store that can hold them declares `cultcache.store.v3`; TypeScript,
|
Element ids: a store that can hold them declares `cultcache.store.v3`; TypeScript,
|
||||||
Python and Rust read it and keep the header, and readers older than ids refuse it
|
Python and Rust read it and keep the header, and the C# runtime decides the mark from content, and these runtimes never drop it. A C# reader from `cultlib-unity-v2.0.0` on refuses a v3 single-file store it cannot read (earlier pins do not)
|
||||||
(vectors: `tests/vectors/document-variants-c2a`).
|
(vectors: `tests/vectors/document-variants-c2a`).
|
||||||
Document variants: C# resolves them (`cultcache.store.v2`); every other runtime,
|
Document variants: C# resolves them (`cultcache.store.v2`); every other runtime,
|
||||||
CultMesh single-file reads, the directory store and CultNet refuse a variant by name
|
CultMesh single-file reads, the directory store and CultNet refuse a variant by name
|
||||||
|
|
|
||||||
|
|
@ -167,8 +167,8 @@ fn first_call() -> anyhow::Result<()> {
|
||||||
|
|
||||||
## Python first call
|
## Python first call
|
||||||
|
|
||||||
A document is a definition with an encoder and decoder; `name=` and `indexes=` register the name lookup
|
A document is a definition with an encoder and decoder; `name=` registers the name lookup, `indexes=` unique lookups and
|
||||||
and indexes. The source is
|
`roles=` many-row selection aliases. The source is
|
||||||
[`packages/cultcache-py/tests/test_first_call.py`](../../packages/cultcache-py/tests/test_first_call.py),
|
[`packages/cultcache-py/tests/test_first_call.py`](../../packages/cultcache-py/tests/test_first_call.py),
|
||||||
run by `python -m unittest tests.test_first_call` in `packages/cultcache-py` with `src` on `PYTHONPATH`
|
run by `python -m unittest tests.test_first_call` in `packages/cultcache-py` with `src` on `PYTHONPATH`
|
||||||
and `msgpack` installed (the store needs it). A second test fails if the block below stops matching the
|
and `msgpack` installed (the store needs it). A second test fails if the block below stops matching the
|
||||||
|
|
@ -194,7 +194,6 @@ item_doc = define_document_type(
|
||||||
encode=lambda item: asdict(item),
|
encode=lambda item: asdict(item),
|
||||||
decode=lambda raw: Item(**raw),
|
decode=lambda raw: Item(**raw),
|
||||||
name="name",
|
name="name",
|
||||||
indexes={"category": "category"},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,13 +33,16 @@ Probes: scratch tests in this worktree, removed. C# filtered runs on Starfire (o
|
||||||
|
|
||||||
| Probe | C# | TypeScript | Rust | Python |
|
| Probe | C# | TypeScript | Rust | Python |
|
||||||
|---|---|---|---|---|
|
|---|---|---|---|---|
|
||||||
| P1 cache {A,B}: B at k, then A at k | **C**: B's record replaced, k now A's id | C, and **both** records live at k | C, both live at k | C, both live at k |
|
| P1 cache {A,B}: B at k, then A at k | **C**: B's record replaced, k now A's id | R `SchemaConflictError` (cultcache-ts: one record per key) | R `SchemaConflictError` naming k, A not written (leaf cut) | R `SchemaConflictError`, A's write refused, B kept (was C, both live) |
|
||||||
| P2 cache {A} only, store holds B at k, write A at k | R `CultSchemaConflictException` (B is foreign, carried) | R at load: "No schema is registered for persisted schema id" | R at load, same | R at load: "Unknown persisted document type" |
|
| P2 cache {A} only, store holds B at k, write A at k | R `CultSchemaConflictException` (B is foreign, carried) | R `SchemaConflictError` at write (foreign loads, carried; cultcache-ts) | R `SchemaConflictError` (B loads as foreign, carried; leaf cut) | R `SchemaConflictError` (B is foreign, carried; the load admits it; was R at load) |
|
||||||
| P3 cacheless `WriteDocument<A>` over B, registry null / {A} / {A,B} | R / R / R | no cacheless write | no cacheless write | no cacheless write |
|
| P3 cacheless `WriteDocument<A>` over B, registry null / {A} / {A,B} | R / R / R | no cacheless write | no cacheless write | no cacheless write |
|
||||||
| P4 cache {A,B}: remove k and upsert A at k in one batch | C (k is A's) | n/a | n/a | n/a |
|
| P4 cache {A,B}: remove k and upsert A at k in one batch | C (k is A's) | n/a | n/a | n/a |
|
||||||
| P5 cache {A,B}: remove, then add in a second commit | C | n/a | n/a | n/a |
|
| P5 cache {A,B}: remove, then add in a second commit | C | n/a | n/a | n/a |
|
||||||
| P6 N2 over an N1 record (undeclared name drift), cache {N2} / cacheless reg {N2} | C restamped / C restamped | not probed | not probed | C (cache) |
|
| P6 N2 over an N1 record (undeclared name drift), cache {N2} / cacheless reg {N2} | C restamped / C restamped | not probed | not probed | C (cache) |
|
||||||
|
|
||||||
|
The C#-written probe the leaves are held to is `tests/vectors/key-identity` (12 rows: retype, foreign carry, key in two stores), walked by
|
||||||
|
`KeyIdentityVectorTests`; each leaf cut corrects its own column above.
|
||||||
|
|
||||||
What this establishes:
|
What this establishes:
|
||||||
|
|
||||||
- C#: one record per key. The only path that retypes is the cache arm (P1): `CacheBackingStore.RefuseUndeclared`
|
- C#: one record per key. The only path that retypes is the cache arm (P1): `CacheBackingStore.RefuseUndeclared`
|
||||||
|
|
@ -52,12 +55,22 @@ What this establishes:
|
||||||
- Retype as remove plus add already works in the cache (P4, P5); the cut adds no new primitive.
|
- Retype as remove plus add already works in the cache (P4, P5); the cut adds no new primitive.
|
||||||
- Directory store: its own judge (`DirectoryMessagePackBackingStore.cs:287-306`) asks `Registry.Owns` for every key,
|
- Directory store: its own judge (`DirectoryMessagePackBackingStore.cs:287-306`) asks `Registry.Owns` for every key,
|
||||||
upserts included. It must take the same per-key rule.
|
upserts included. It must take the same per-key rule.
|
||||||
- TypeScript, Rust, Python: records are keyed by `(type, key)` (Rust `entries: BTreeMap<(String, String), _>`,
|
- TypeScript and Python: records are keyed by `(type, key)` (TS `#entryId(definition.type, key)`, `cult-cache.ts:323`;
|
||||||
`lib.rs:2659`; TS `#entryId(definition.type, key)`, `cult-cache.ts:323`; Python sorts and stores by `(type, key)`).
|
Python sorts and stores by `(type, key)`). A write of A at k never meets B's record: both live at k (P1). Neither carries
|
||||||
A write of A at k never meets B's record: both live at k (P1). None carries a foreign record; a store holding an
|
a foreign record; a store holding an unregistered type does not load (P2). So neither has a "write meets another type's
|
||||||
unregistered type does not load (P2). So no leaf has a "write meets another type's record" path to fix. Bringing the
|
record" path to fix: bringing the ruling to them first needs C#'s key identity (one record per key) and C#'s foreign
|
||||||
ruling to the leaves first needs C#'s key identity (one record per key) and C#'s foreign carry, which are parity changes
|
carry, parity changes of their store model, not a write-ownership rule (follow-up `variants:follow_up:leaf-key-identity-parity`).
|
||||||
of the leaves' store model, not a write-ownership rule. Owned by the new follow-up `variants:follow_up:leaf-key-identity-parity`.
|
- Rust is that follow-up's first leaf cut (`cut-leaf-key-identity-rs`). The cache keeps its `(type, key)` map; one judge
|
||||||
|
(`refuse_retype`) refuses another registered type at a key the cache holds, or any write at a key a store carries
|
||||||
|
foreign, on every write path, pull and SoA load, comparing types, not the ids envelopes claim. At a key the cache does not
|
||||||
|
hold (a fresh key, or one another writer filled after this cache's pull) the single-file store's commit guard judges by
|
||||||
|
the stored and claimed schema ids, so an envelope claiming another registered type's id is refused earlier, at every
|
||||||
|
envelope admission (`refuse_claimed_id`, the pull's own `resolve_registered_type` lookup); an id naming no registered type
|
||||||
|
passes, trusted as claimed: an envelope under such an id can replace another writer's record stored under it at a key
|
||||||
|
this cache does not hold, or be served as another type by a reader that registers the type the id names (Soul g8a, g8c,
|
||||||
|
open: which ids a Rust type owns is not declared to CultCache). Store-level writers called directly (`push`,
|
||||||
|
`insert_entry_if_absent`, compare-and-swap) trust the claimed id.
|
||||||
|
The cache carries an unregistered type's records as foreign and walks the vector's 12 rows.
|
||||||
- Interop (unproven): a Python store holding A and B at k fails C# decode first on payload encoding (Python's
|
- Interop (unproven): a Python store holding A and B at k fails C# decode first on payload encoding (Python's
|
||||||
lambda-defined document wrote JSON payloads), so what C# does with two records at one key was not reached.
|
lambda-defined document wrote JSON payloads), so what C# does with two records at one key was not reached.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,18 @@
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name.
|
- A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`.
|
||||||
|
|
||||||
|
- A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it.
|
||||||
|
- Added `foreign_records()` and the frozen `ForeignRecord(key, schema_id, schema_name, stored_at)`, sorted by key in UTF-16 code-unit order. `stores.retyped` and `stores.written_owns` lose their underscore: the cache judge and the single-file store's commit guard ask one predicate.
|
||||||
|
|
||||||
|
- A single-file store holding two records under one key is unreadable: the v1 reader and the legacy envelope-array reader refuse it with `StoreUnreadableError` (a key is compared by its exact code points, no case folding or normalisation). A catalog slot whose canonical schema is not a string is refused the same way. A store that loaded before can now be refused: repair it with the runtime that wrote the duplicate.
|
||||||
|
- A `put` or `push_all` that would land a second type under a key the store holds is refused with `SchemaConflictError`, and the file is left as it was. Retype a key by deleting it, then putting the new type. A write replaces the record at a key only when the written type owns it: its catalog entry declares the stored schema id (its own, or one it lists compatible), so a renamed type that lists the old id compatible may replace the record, or it has the stored record's schema name. This runtime does not compare shapes, so a same-name type is never refused for dropping persisted slots as C# refuses it.
|
||||||
|
- Writes of the same type replace by key: the last record wins, in a batch as in the file.
|
||||||
|
|
||||||
|
- `indexes` are unique, as C# `[CultIndex]`: a write, load or `register_index` that leaves two records of a type holding one non-blank value under one alias is refused (`CultCacheError`) naming both keys and the alias, and writes nothing. Index text is the C# canonical text (`True`/`False`, integers as digits, floats as the exact expansion, blank values never indexed), so an empty or whitespace string is no longer a value. A new `roles` map names the many-row selection aliases (never indexed for lookup, same catalog `index_alias`). Classify every `indexes` key per the README before updating. The verdict is the contract, the text is informational.
|
||||||
|
|
||||||
|
- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
||||||
- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `push_all` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again.
|
- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `push_all` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again.
|
||||||
- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a
|
- Added `StoreUnreadableError` (a `ValueError`) for a store file this runtime cannot read, and `SchemaConflictError` (a `ValueError`) for a
|
||||||
write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload.
|
write the catalog cannot describe. A store is exactly three slots; a schema is never recovered from a record's payload.
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,11 @@ discovery, sessions, local server) build on it, mirroring `cultcache-rs` /
|
||||||
`[format, schemaCatalog, records]`
|
`[format, schemaCatalog, records]`
|
||||||
- records store raw MessagePack payload bytes under schema-catalog identity
|
- records store raw MessagePack payload bytes under schema-catalog identity
|
||||||
- payloads decode only through the registered document definition for that type
|
- payloads decode only through the registered document definition for that type
|
||||||
- unknown persisted types fail closed
|
- a cache holds one record per key whatever its type: a write of a second type at a held key is refused with
|
||||||
|
`SchemaConflictError` (in memory too), and two stores holding one key refuse the load
|
||||||
|
- a store record no registered type claims loads and is carried as foreign: `cache.foreign_records()` lists it, the store
|
||||||
|
keeps its bytes through every write, a write or removal at its key is refused, and registering a type that declares its
|
||||||
|
schema id claims it at the next load
|
||||||
- global documents are singleton-style per type
|
- global documents are singleton-style per type
|
||||||
- one home store per document type: a store added with `add_backing_store`
|
- one home store per document type: a store added with `add_backing_store`
|
||||||
owns its types, otherwise the generic store is the home; a type claimed by a
|
owns its types, otherwise the generic store is the home; a type claimed by a
|
||||||
|
|
@ -73,7 +77,7 @@ store.
|
||||||
- `register_document_type(...)`
|
- `register_document_type(...)`
|
||||||
- `register_registry(...)`
|
- `register_registry(...)`
|
||||||
- `register_name_lookup(...)`
|
- `register_name_lookup(...)`
|
||||||
- `register_index(...)`
|
- `register_index(...)` (a unique index)
|
||||||
- `add_backing_store(...)`
|
- `add_backing_store(...)`
|
||||||
- `add_generic_store(...)`
|
- `add_generic_store(...)`
|
||||||
- `pull_all_backing_stores()`
|
- `pull_all_backing_stores()`
|
||||||
|
|
@ -93,8 +97,36 @@ store.
|
||||||
- `update_global(...)`
|
- `update_global(...)`
|
||||||
- `delete(...)`
|
- `delete(...)`
|
||||||
- `delete_global(...)`
|
- `delete_global(...)`
|
||||||
|
- `is_writable(document, key)` (advisory: would a write or removal of the key be refused; see below)
|
||||||
- `snapshot()`
|
- `snapshot()`
|
||||||
|
|
||||||
|
## Indexes And Roles
|
||||||
|
|
||||||
|
`indexes=` are unique lookups (variants:ruling:unique-indexes; C# `[CultIndex]`): within one document type and one alias
|
||||||
|
no two records hold one non-blank value, judged on the result of every write (a batch is judged whole, so a swap inside
|
||||||
|
one `put_envelopes` lands) and of every load and `register_index`. A refusal is a `CultCacheError`,
|
||||||
|
`Records <k1> and <k2> (<schema name>) both hold unique index '<alias>'; an index value names one record.`, keys in
|
||||||
|
ordinal order, never the value; it writes nothing, and a refused load or registration installs nothing. The contract is
|
||||||
|
the verdict; the text is informational. Blank values (absent, empty, or only C# `White_Space` characters) are never
|
||||||
|
indexed and never conflict. Two types may share an alias and a value. Values compare as the C# canonical text: strings as
|
||||||
|
is (never trimmed), `True`/`False`, integers as digits, a non-integral float as its exact decimal expansion, NaN and
|
||||||
|
infinities no value.
|
||||||
|
|
||||||
|
`roles=` name selection aliases (C# `[CultRole]`): many records may share a value, nothing is indexed or refused,
|
||||||
|
`get_by_index` on a role answers `None`, and the member's catalog `index_alias` is the same either way, so catalog bytes
|
||||||
|
do not move. An alias is declared in one of the two. `CultMeshDatabase.watch_by_index` reads `indexes` only.
|
||||||
|
|
||||||
|
Migrating an existing `indexes` key, per key, before bumping:
|
||||||
|
|
||||||
|
1. Nothing looks records up by the key (`get_by_index`, `get_key_by_index`, `node.get_by_index`, `watch_by_index`): move it to `roles`.
|
||||||
|
2. Something looks it up: it stays unique only when two live records can never hold one non-blank value (the record key
|
||||||
|
is derived from that value alone, or it is an outside identity: email, sku, external id). Enums, statuses, kinds,
|
||||||
|
categories, parent or foreign ids, shared hashes and timestamps are roles; select them with `get_all` plus a filter.
|
||||||
|
3. Open each real `.cc` once with the new definitions. A refusal names both keys and the alias and writes nothing.
|
||||||
|
|
||||||
|
Misclassifying: a role declared unique makes a store that holds a shared value refuse to open until the key is
|
||||||
|
reclassified; a unique key declared a role answers no lookups and stops refusing duplicates.
|
||||||
|
|
||||||
## Persistence Model
|
## Persistence Model
|
||||||
|
|
||||||
Backing stores persist envelopes, not caller domain objects. The cache decodes payloads through registered document definitions and rejects unknown type discriminators. This keeps Python's dynamic runtime from turning persistence into an open polymorphic sewer with a cheerful docstring.
|
Backing stores persist envelopes, not caller domain objects. The cache decodes payloads through registered document definitions and rejects unknown type discriminators. This keeps Python's dynamic runtime from turning persistence into an open polymorphic sewer with a cheerful docstring.
|
||||||
|
|
@ -112,6 +144,10 @@ used by the TypeScript, Rust, and C# runtimes:
|
||||||
|
|
||||||
Large corpora should use a sharded store or a real database. A single snapshot file is a scalpel, not a forklift.
|
Large corpora should use a sharded store or a real database. A single snapshot file is a scalpel, not a forklift.
|
||||||
|
|
||||||
|
## Store format and rollback
|
||||||
|
|
||||||
|
A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-py from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.)
|
||||||
|
|
||||||
## DatabaseEntry Slot Contract
|
## DatabaseEntry Slot Contract
|
||||||
|
|
||||||
For cross-runtime cache entries, use `define_database_entry_type(...)` instead
|
For cross-runtime cache entries, use `define_database_entry_type(...)` instead
|
||||||
|
|
@ -145,6 +181,16 @@ The payload is a MessagePack array. Field keys are durable slot indexes:
|
||||||
- newly added fields should use new keys
|
- newly added fields should use new keys
|
||||||
- fields with defaults tolerate missing or nil slots when older payloads are read
|
- fields with defaults tolerate missing or nil slots when older payloads are read
|
||||||
|
|
||||||
|
A payload carrying a slot past the slots a reader declares, at any depth (the document, a nested record, a list element, a
|
||||||
|
union arm, a unit arm), was written by a newer build. It is read, and the record is held read-only: `put`, `update`,
|
||||||
|
`put_envelope(s)`, `delete` and their global forms refuse its key with `SchemaConflictError` before any store is touched, and
|
||||||
|
`SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` naming the
|
||||||
|
first type read with extra slots (never a slot value). `CultCache.is_writable(document, key)` answers the same question before
|
||||||
|
the write: False for such a record, for a key a store carries foreign, and for a key held as another type's record; the write
|
||||||
|
still decides. `DocumentDefinition.decode_payload_marked(payload)` returns the value and its `UnreadSlots` (or None). Declaring the
|
||||||
|
stored id in `compatible_schema_ids` consents to the loss of depth-0 slots that id's catalog entry reports; a nested mark never is.
|
||||||
|
A store keeps a read-only record's bytes through every other write. A document with a custom `decode` never marks.
|
||||||
|
|
||||||
That matches the Rust `#[derive(DatabaseEntry)]` formatter shape and the C#
|
That matches the Rust `#[derive(DatabaseEntry)]` formatter shape and the C#
|
||||||
`[Key(n)]` intent: schema evolution comes from stable field slots, not source
|
`[Key(n)]` intent: schema evolution comes from stable field slots, not source
|
||||||
member order.
|
member order.
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
from .backing_store import BackingStore, CultCacheEnvelope
|
from .backing_store import BackingStore, CultCacheEnvelope
|
||||||
from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember
|
from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember
|
||||||
from . import leaves
|
from . import leaves
|
||||||
from .cache import CultCache, CultCacheBuilder
|
from .cache import CultCache, CultCacheBuilder, ForeignRecord
|
||||||
from .documents import (
|
from .documents import (
|
||||||
MAX_PAYLOAD_DEPTH,
|
MAX_PAYLOAD_DEPTH,
|
||||||
MAX_SLOT_KEY,
|
MAX_SLOT_KEY,
|
||||||
|
|
@ -9,6 +9,7 @@ from .documents import (
|
||||||
DatabaseEntryField,
|
DatabaseEntryField,
|
||||||
DocumentDefinition,
|
DocumentDefinition,
|
||||||
UnionArm,
|
UnionArm,
|
||||||
|
UnreadSlots,
|
||||||
database_entry_field,
|
database_entry_field,
|
||||||
define_database_entry_type,
|
define_database_entry_type,
|
||||||
define_document_registry,
|
define_document_registry,
|
||||||
|
|
@ -29,6 +30,7 @@ __all__ = [
|
||||||
"CultCacheSchemaCatalogMember",
|
"CultCacheSchemaCatalogMember",
|
||||||
"DatabaseEntryField",
|
"DatabaseEntryField",
|
||||||
"DocumentDefinition",
|
"DocumentDefinition",
|
||||||
|
"ForeignRecord",
|
||||||
"JsonLinesBackingStore",
|
"JsonLinesBackingStore",
|
||||||
"MAX_PAYLOAD_DEPTH",
|
"MAX_PAYLOAD_DEPTH",
|
||||||
"MAX_SLOT_KEY",
|
"MAX_SLOT_KEY",
|
||||||
|
|
@ -37,6 +39,7 @@ __all__ = [
|
||||||
"SlotCodec",
|
"SlotCodec",
|
||||||
"StoreUnreadableError",
|
"StoreUnreadableError",
|
||||||
"UnionArm",
|
"UnionArm",
|
||||||
|
"UnreadSlots",
|
||||||
"leaves",
|
"leaves",
|
||||||
"database_entry_field",
|
"database_entry_field",
|
||||||
"define_database_entry_type",
|
"define_database_entry_type",
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,15 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
|
import math
|
||||||
import threading
|
import threading
|
||||||
|
from decimal import Decimal
|
||||||
from dataclasses import dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
from typing import Any, Callable, Generic, TypeVar
|
from typing import Any, Callable, Generic, Iterable, TypeVar
|
||||||
|
|
||||||
from .backing_store import BackingStore, CultCacheEnvelope
|
from .backing_store import BackingStore, CultCacheEnvelope
|
||||||
from .documents import DocumentDefinition, extract_value
|
from .documents import DocumentDefinition, UnreadSlots, extract_value
|
||||||
from .stores import SchemaConflictError
|
from .stores import SchemaConflictError, retyped, written_owns
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
F = TypeVar("F", bound=Callable[..., Any])
|
F = TypeVar("F", bound=Callable[..., Any])
|
||||||
|
|
@ -52,6 +54,52 @@ class CultCacheError(RuntimeError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# Blank is no value: empty or only C# White_Space characters (the 25 chars string.IsNullOrWhiteSpace is true for).
|
||||||
|
# An explicit set, not str.isspace(): Python also counts U+001C-001F, which C# does not, and omits nothing C# counts.
|
||||||
|
_BLANK_INDEX_CHARS = frozenset(
|
||||||
|
chr(code)
|
||||||
|
for code in (
|
||||||
|
*range(0x09, 0x0E),
|
||||||
|
0x20,
|
||||||
|
0x85,
|
||||||
|
0xA0,
|
||||||
|
0x1680,
|
||||||
|
*range(0x2000, 0x200B),
|
||||||
|
0x2028,
|
||||||
|
0x2029,
|
||||||
|
0x202F,
|
||||||
|
0x205F,
|
||||||
|
0x3000,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _utf16_order(text: str) -> bytes:
|
||||||
|
"""Sorts as C# StringComparer.Ordinal does: by UTF-16 code unit, not code point."""
|
||||||
|
return text.encode("utf-16-be", "surrogatepass")
|
||||||
|
|
||||||
|
|
||||||
|
def index_text(value: Any) -> str | None:
|
||||||
|
"""The canonical text of an index value, as C# IndexValue and RenderCanonicalNumber, or None when it names no
|
||||||
|
value: a string is as is (never trimmed), a boolean is `True`/`False`, an integer is its digits, a non-integral
|
||||||
|
finite float is its exact decimal expansion, NaN and infinities and blank strings are no value."""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, bool): # before int: bool is an int subclass
|
||||||
|
return "True" if value else "False"
|
||||||
|
if isinstance(value, int):
|
||||||
|
return str(value)
|
||||||
|
if isinstance(value, float):
|
||||||
|
if not math.isfinite(value):
|
||||||
|
return None
|
||||||
|
if value.is_integer():
|
||||||
|
return str(int(value)) # -0.0 is 0
|
||||||
|
text = format(Decimal(value), "f") # Decimal(float) is exact
|
||||||
|
return text.rstrip("0").rstrip(".") if "." in text else text
|
||||||
|
text = value if isinstance(value, str) else str(value)
|
||||||
|
return None if all(char in _BLANK_INDEX_CHARS for char in text) else text
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class _State:
|
class _State:
|
||||||
documents: dict[str, DocumentDefinition[Any]] = field(default_factory=dict)
|
documents: dict[str, DocumentDefinition[Any]] = field(default_factory=dict)
|
||||||
|
|
@ -72,6 +120,24 @@ class _State:
|
||||||
# of that global removes it from the home store. Remove once no store holds a global under a key
|
# of that global removes it from the home store. Remove once no store holds a global under a key
|
||||||
# other than GLOBAL_KEY.
|
# other than GLOBAL_KEY.
|
||||||
legacy_global_keys: dict[str, str] = field(default_factory=dict)
|
legacy_global_keys: dict[str, str] = field(default_factory=dict)
|
||||||
|
# The records the last pull read that no registered type claims, sorted by key. Derived from the pull and swapped with the
|
||||||
|
# rest of the state, written nowhere else; it decides only refusals at its keys. A store keeps their bytes.
|
||||||
|
foreign: list["ForeignRecord"] = field(default_factory=list)
|
||||||
|
# The records held read-only, by key (a key holds one record) with the type that holds it: the payload carried slots past the
|
||||||
|
# declared ones and nothing consents to the loss. Derived from the load or the write that admitted the record and swapped with
|
||||||
|
# the rest of the state; it decides only refusals at its keys, and any other key is absent.
|
||||||
|
unread: dict[str, tuple[str, UnreadSlots]] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ForeignRecord:
|
||||||
|
"""A record a store holds that no registered type claims: the cache does not serve it, refuses a write or a removal at its
|
||||||
|
key, and a store keeps its bytes. Registering a type that declares its schema id claims it at the next load."""
|
||||||
|
|
||||||
|
key: str
|
||||||
|
schema_id: str
|
||||||
|
schema_name: str
|
||||||
|
stored_at: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|
@ -186,6 +252,8 @@ class CultCache:
|
||||||
@_mutating
|
@_mutating
|
||||||
def register_index(self, document: DocumentDefinition[Any], index: str, extractor: str | Any) -> None:
|
def register_index(self, document: DocumentDefinition[Any], index: str, extractor: str | Any) -> None:
|
||||||
self._assert_registered(document)
|
self._assert_registered(document)
|
||||||
|
if index in document.roles:
|
||||||
|
raise CultCacheError(f"Index '{index}' of {document.type} is declared as a role; a role is never a unique index.")
|
||||||
index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()}
|
index_extractors = {type: dict(indexes) for type, indexes in self._state.index_extractors.items()}
|
||||||
index_extractors.setdefault(document.type, {})[index] = extractor
|
index_extractors.setdefault(document.type, {})[index] = extractor
|
||||||
self._install_lookups(self._state.name_extractors, index_extractors)
|
self._install_lookups(self._state.name_extractors, index_extractors)
|
||||||
|
|
@ -195,7 +263,11 @@ class CultCache:
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Derives every lookup under the candidate extractors, then installs extractors and lookups
|
"""Derives every lookup under the candidate extractors, then installs extractors and lookups
|
||||||
together. An extractor that raises on a held value installs nothing."""
|
together. An extractor that raises on a held value installs nothing."""
|
||||||
names, indexes, lookup_keys = self._derive_lookups(self._state.values, name_extractors, index_extractors)
|
names, indexes, lookup_keys = self._derive_lookups(
|
||||||
|
[(type, key, value) for type, values in self._state.values.items() for key, value in values.items()],
|
||||||
|
name_extractors,
|
||||||
|
index_extractors,
|
||||||
|
)
|
||||||
(
|
(
|
||||||
self._state.name_extractors,
|
self._state.name_extractors,
|
||||||
self._state.index_extractors,
|
self._state.index_extractors,
|
||||||
|
|
@ -263,19 +335,29 @@ class CultCache:
|
||||||
structures, running every check and every user extractor; the cache's state is replaced
|
structures, running every check and every user extractor; the cache's state is replaced
|
||||||
only once that build finishes, so a refused load admits nothing."""
|
only once that build finishes, so a refused load admits nothing."""
|
||||||
loaded_values: dict[str, dict[str, Any]] = {}
|
loaded_values: dict[str, dict[str, Any]] = {}
|
||||||
|
read_values: dict[tuple[str, str], Any] = {} # (type, key) -> value, one per key, the last read wins; the judge is order-free
|
||||||
loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {}
|
loaded_envelopes: dict[str, dict[str, CultCacheEnvelope]] = {}
|
||||||
loaded_legacy_global_keys: dict[str, str] = {}
|
loaded_legacy_global_keys: dict[str, str] = {}
|
||||||
routes, generic = self._state.stores_by_type, self._state.generic_store
|
routes, generic = self._state.stores_by_type, self._state.generic_store
|
||||||
seen_globals: set[str] = set()
|
seen_globals: set[str] = set()
|
||||||
|
foreign: list[ForeignRecord] = []
|
||||||
|
loaded_unread: dict[str, tuple[str, UnreadSlots]] = {}
|
||||||
|
type_at_key: dict[str, tuple[str, int]] = {} # one record per key across every store: its type (or schema name) and store
|
||||||
stores: list[BackingStore] = []
|
stores: list[BackingStore] = []
|
||||||
for candidate in [*self._state.stores_by_type.values(), self._state.generic_store]:
|
for candidate in [*self._state.stores_by_type.values(), self._state.generic_store]:
|
||||||
if candidate is not None and all(candidate is not store for store in stores):
|
if candidate is not None and all(candidate is not store for store in stores):
|
||||||
stores.append(candidate)
|
stores.append(candidate)
|
||||||
for store in stores:
|
for position, store in enumerate(stores):
|
||||||
for envelope in store.pull_all():
|
for envelope in store.pull_all():
|
||||||
document = self._resolve_document_for_envelope(envelope)
|
document = self._resolve_document_for_envelope(envelope)
|
||||||
if document is None:
|
if document is None:
|
||||||
raise CultCacheError(f"Unknown persisted document type: {envelope.type}")
|
entry = envelope.catalog_entry
|
||||||
|
held_name = entry.schema_name if entry is not None else envelope.type
|
||||||
|
self._claim_key(type_at_key, envelope.key, held_name, position)
|
||||||
|
foreign.append(
|
||||||
|
ForeignRecord(envelope.key, envelope.schema_id or envelope.type, held_name, envelope.stored_at)
|
||||||
|
)
|
||||||
|
continue
|
||||||
if envelope.type != document.type:
|
if envelope.type != document.type:
|
||||||
envelope = CultCacheEnvelope(
|
envelope = CultCacheEnvelope(
|
||||||
key=envelope.key,
|
key=envelope.key,
|
||||||
|
|
@ -298,12 +380,16 @@ class CultCache:
|
||||||
if envelope.key != GLOBAL_KEY:
|
if envelope.key != GLOBAL_KEY:
|
||||||
loaded_legacy_global_keys[envelope.type] = envelope.key
|
loaded_legacy_global_keys[envelope.type] = envelope.key
|
||||||
envelope = replace(envelope, key=GLOBAL_KEY)
|
envelope = replace(envelope, key=GLOBAL_KEY)
|
||||||
envelope = self._stamped(document, envelope)
|
self._claim_key(type_at_key, envelope.key, envelope.type, position)
|
||||||
value = document.decode_payload(envelope.payload)
|
arrived, envelope = envelope, self._stamped(document, envelope)
|
||||||
|
value, unread = self._decode_marked(document, arrived)
|
||||||
|
if unread is not None:
|
||||||
|
loaded_unread[envelope.key] = (envelope.type, unread)
|
||||||
loaded_values.setdefault(envelope.type, {})[envelope.key] = value
|
loaded_values.setdefault(envelope.type, {})[envelope.key] = value
|
||||||
|
read_values[(envelope.type, envelope.key)] = value
|
||||||
loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope
|
loaded_envelopes.setdefault(envelope.type, {})[envelope.key] = envelope
|
||||||
loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups(
|
loaded_names, loaded_indexes, loaded_lookup_keys = self._derive_lookups(
|
||||||
loaded_values, self._state.name_extractors, self._state.index_extractors
|
[(type, key, value) for (type, key), value in read_values.items()], self._state.name_extractors, self._state.index_extractors
|
||||||
)
|
)
|
||||||
# Plain attribute assignments: nothing fallible runs once the swap begins.
|
# Plain attribute assignments: nothing fallible runs once the swap begins.
|
||||||
(
|
(
|
||||||
|
|
@ -313,7 +399,30 @@ class CultCache:
|
||||||
self._state.indexes,
|
self._state.indexes,
|
||||||
self._state.lookup_keys,
|
self._state.lookup_keys,
|
||||||
self._state.legacy_global_keys,
|
self._state.legacy_global_keys,
|
||||||
) = (loaded_values, loaded_envelopes, loaded_names, loaded_indexes, loaded_lookup_keys, loaded_legacy_global_keys)
|
self._state.foreign,
|
||||||
|
self._state.unread,
|
||||||
|
) = (
|
||||||
|
loaded_values,
|
||||||
|
loaded_envelopes,
|
||||||
|
loaded_names,
|
||||||
|
loaded_indexes,
|
||||||
|
loaded_lookup_keys,
|
||||||
|
loaded_legacy_global_keys,
|
||||||
|
sorted(foreign, key=lambda record: _utf16_order(record.key)),
|
||||||
|
loaded_unread,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _claim_key(type_at_key: dict[str, tuple[str, int]], key: str, type: str, store: int) -> None:
|
||||||
|
"""A key holds one record whatever its type, across every store a load reads. The refusal names the key and both types,
|
||||||
|
never a payload."""
|
||||||
|
held, held_store = type_at_key.setdefault(key, (type, store))
|
||||||
|
if held != type or held_store != store:
|
||||||
|
raise CultCacheError(f"Record {key!r} is held by both {held} and {type}: a key holds one record; the load admitted nothing.")
|
||||||
|
|
||||||
|
def foreign_records(self) -> list[ForeignRecord]:
|
||||||
|
"""The records the last load read that no registered type claims, sorted by key (UTF-16 code-unit order)."""
|
||||||
|
return list(self._state.foreign)
|
||||||
|
|
||||||
def get(self, document: DocumentDefinition[T], key: str) -> T | None:
|
def get(self, document: DocumentDefinition[T], key: str) -> T | None:
|
||||||
self._assert_registered(document)
|
self._assert_registered(document)
|
||||||
|
|
@ -391,9 +500,9 @@ class CultCache:
|
||||||
def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T:
|
def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T:
|
||||||
self._assert_registered(document)
|
self._assert_registered(document)
|
||||||
self._check_envelope(document, envelope)
|
self._check_envelope(document, envelope)
|
||||||
envelope = self._stamped(document, envelope)
|
arrived, envelope = envelope, self._stamped(document, envelope)
|
||||||
value = document.decode_payload(envelope.payload)
|
value, unread = self._decode_marked(document, envelope)
|
||||||
self._write(document, [(envelope, value)], batch=False)
|
self._write(document, [(envelope, value)], batch=False, unread={envelope.key: unread})
|
||||||
return value
|
return value
|
||||||
|
|
||||||
@_mutating
|
@_mutating
|
||||||
|
|
@ -401,13 +510,33 @@ class CultCache:
|
||||||
self._assert_registered(document)
|
self._assert_registered(document)
|
||||||
values: list[T] = []
|
values: list[T] = []
|
||||||
stamped: list[CultCacheEnvelope] = []
|
stamped: list[CultCacheEnvelope] = []
|
||||||
|
unread: dict[str, UnreadSlots | None] = {}
|
||||||
for envelope in envelopes:
|
for envelope in envelopes:
|
||||||
self._check_envelope(document, envelope)
|
self._check_envelope(document, envelope)
|
||||||
stamped.append(self._stamped(document, envelope))
|
stamped.append(self._stamped(document, envelope))
|
||||||
values.append(document.decode_payload(envelope.payload))
|
value, unread[envelope.key] = self._decode_marked(document, stamped[-1])
|
||||||
self._write(document, list(zip(stamped, values)), batch=True)
|
values.append(value)
|
||||||
|
self._write(document, list(zip(stamped, values)), batch=True, unread=unread)
|
||||||
return values
|
return values
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode_marked(document: DocumentDefinition[Any], held_as: CultCacheEnvelope) -> tuple[Any, UnreadSlots | None]:
|
||||||
|
"""The payload decoded under the document, with the mark of any slot it carried past the declared ones, or no mark when the
|
||||||
|
loss is consented. `held_as` is the envelope under the id the record is stored under: a load's arrival, a write's stamped
|
||||||
|
envelope, so a key is writable in memory exactly when it is after a reload. A depth-0 mark is consented only when that id is
|
||||||
|
one the document lists compatible and its catalog entry has slots past the document's that cover the mark's slot count (the
|
||||||
|
loss a compatible-id declaration consents to, cultlib-gaps:ruling:slot-mark-consent). Any other mark keeps the record read-only."""
|
||||||
|
value, slots = document.decode_payload_marked(held_as.payload)
|
||||||
|
if slots is None:
|
||||||
|
return value, None
|
||||||
|
entry, stored = document.catalog_entry(), held_as.catalog_entry
|
||||||
|
if slots.depth0_only and stored is not None and held_as.schema_id in entry.compatible_schema_ids and held_as.schema_id != entry.schema_id:
|
||||||
|
declared = {member.slot for member in entry.members}
|
||||||
|
reported = [member.slot for member in stored.members if member.slot not in declared]
|
||||||
|
if reported and slots.slot_count <= max(reported) + 1:
|
||||||
|
return value, None
|
||||||
|
return value, slots
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _stamped(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> CultCacheEnvelope:
|
def _stamped(document: DocumentDefinition[Any], envelope: CultCacheEnvelope) -> CultCacheEnvelope:
|
||||||
"""The record under its registered schema's id, whatever id the envelope arrived under, so the catalog entry a store
|
"""The record under its registered schema's id, whatever id the envelope arrived under, so the catalog entry a store
|
||||||
|
|
@ -427,17 +556,32 @@ class CultCache:
|
||||||
raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}")
|
raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}")
|
||||||
|
|
||||||
def _write(
|
def _write(
|
||||||
self, document: DocumentDefinition[Any], records: list[tuple[CultCacheEnvelope, Any]], *, batch: bool
|
self,
|
||||||
|
document: DocumentDefinition[Any],
|
||||||
|
records: list[tuple[CultCacheEnvelope, Any]],
|
||||||
|
*,
|
||||||
|
batch: bool,
|
||||||
|
unread: dict[str, UnreadSlots | None] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Validates every record (home and name and index extractors included) before the store is
|
"""Validates every record (home and name and index extractors included) before the store is
|
||||||
touched; once the store accepts, applying to the cache has no fallible step."""
|
touched; once the store accepts, applying to the cache has no fallible step."""
|
||||||
type = document.type
|
type = document.type
|
||||||
|
# A batch naming a key twice lands the last record for it, as C# CultCacheBatch stages one operation per key. The
|
||||||
|
# judge, the store and the apply all read this one list.
|
||||||
|
records = list({envelope.key: (envelope, value) for envelope, value in records}.values())
|
||||||
|
self._refuse_unread([envelope.key for envelope, _ in records])
|
||||||
name_extractor = self._state.name_extractors.get(type)
|
name_extractor = self._state.name_extractors.get(type)
|
||||||
index_extractors = self._state.index_extractors.get(type, {})
|
index_extractors = self._state.index_extractors.get(type, {})
|
||||||
admitted = [
|
admitted = [
|
||||||
(envelope, value, self._lookup_keys_for(value, name_extractor, index_extractors))
|
(envelope, value, self._lookup_keys_for(value, name_extractor, index_extractors))
|
||||||
for envelope, value in records
|
for envelope, value in records
|
||||||
]
|
]
|
||||||
|
displaced = self._refuse_retype(document, [envelope for envelope, _ in records])
|
||||||
|
self._refuse_index_sharing(
|
||||||
|
[(type, envelope.key, keys) for envelope, _, keys in admitted],
|
||||||
|
self._state.indexes,
|
||||||
|
{(type, envelope.key) for envelope, _ in records},
|
||||||
|
)
|
||||||
store = self._store_for_type(type)
|
store = self._store_for_type(type)
|
||||||
legacy_key = self._state.legacy_global_keys.get(type)
|
legacy_key = self._state.legacy_global_keys.get(type)
|
||||||
if store is not None:
|
if store is not None:
|
||||||
|
|
@ -451,16 +595,99 @@ class CultCache:
|
||||||
values = self._state.values.setdefault(type, {})
|
values = self._state.values.setdefault(type, {})
|
||||||
envelopes = self._state.envelopes.setdefault(type, {})
|
envelopes = self._state.envelopes.setdefault(type, {})
|
||||||
lookup_keys = self._state.lookup_keys.setdefault(type, {})
|
lookup_keys = self._state.lookup_keys.setdefault(type, {})
|
||||||
|
for other, key in displaced:
|
||||||
|
self._drop(other, key)
|
||||||
for envelope, value, keys in admitted:
|
for envelope, value, keys in admitted:
|
||||||
self._remove_lookups(type, envelope.key)
|
self._remove_lookups(type, envelope.key)
|
||||||
values[envelope.key] = value
|
values[envelope.key] = value
|
||||||
envelopes[envelope.key] = envelope
|
envelopes[envelope.key] = envelope
|
||||||
|
# A held mark refused this write above, so a key reaching here carries none to clear.
|
||||||
|
if (mark := (unread or {}).get(envelope.key)) is not None:
|
||||||
|
self._state.unread[envelope.key] = (type, mark)
|
||||||
lookup_keys[envelope.key] = keys
|
lookup_keys[envelope.key] = keys
|
||||||
if keys.name is not None:
|
if keys.name is not None:
|
||||||
self._state.names[(type, keys.name)] = envelope.key
|
self._state.names[(type, keys.name)] = envelope.key
|
||||||
for index, index_value in keys.indexes.items():
|
for index, index_value in keys.indexes.items():
|
||||||
self._state.indexes[(type, index, index_value)] = envelope.key
|
self._state.indexes[(type, index, index_value)] = envelope.key
|
||||||
|
|
||||||
|
def _refuse_retype(self, document: DocumentDefinition[Any], written: list[CultCacheEnvelope]) -> list[tuple[str, str]]:
|
||||||
|
"""The one key-identity judge (variants:ruling:cache-refuses-retype): a key holds one record whatever its type, so a write
|
||||||
|
may not land at a key this cache holds another type's record at, unless the written type owns that record's schema id
|
||||||
|
(the single-file store's commit guard asks the same question of the file), and may not land at a key a store carries
|
||||||
|
foreign. Returns the (type, key) records an owning write replaces; refuses, changing nothing, otherwise."""
|
||||||
|
replaced: list[tuple[str, str]] = []
|
||||||
|
foreign = {record.key: record for record in self._state.foreign}
|
||||||
|
for envelope in written:
|
||||||
|
carried = foreign.get(envelope.key)
|
||||||
|
if carried is not None:
|
||||||
|
raise self._foreign_refusal(carried)
|
||||||
|
for other, held_envelopes in self._state.envelopes.items():
|
||||||
|
held = held_envelopes.get(envelope.key)
|
||||||
|
if other == document.type or held is None:
|
||||||
|
continue
|
||||||
|
if not written_owns(held, envelope) or self._store_for_type(other) is not self._store_for_type(document.type):
|
||||||
|
raise retyped(held, envelope)
|
||||||
|
replaced.append((other, envelope.key))
|
||||||
|
return replaced
|
||||||
|
|
||||||
|
@_locked
|
||||||
|
def is_writable(self, document: DocumentDefinition[Any], key: str) -> bool:
|
||||||
|
"""Whether a write or removal of the key by this document would pass the judges that run before any store is touched: False for
|
||||||
|
a record held read-only (its payload carried slots past the declared ones, at any depth, and nothing consents to the loss, so
|
||||||
|
writing it would drop data a newer writer stored), for a key a store carries foreign, and for a key held as another type's
|
||||||
|
record this document does not own. Advisory (the write still decides): it answers at call time, and a key nothing holds is
|
||||||
|
writable."""
|
||||||
|
self._assert_registered(document)
|
||||||
|
try:
|
||||||
|
self._refuse_at(document, key)
|
||||||
|
except (SchemaConflictError, CultCacheError):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _refuse_at(self, document: DocumentDefinition[Any], key: str) -> list[tuple[str, str]]:
|
||||||
|
"""The judges a write or removal of the key by this document passes before any store is touched, asked by `is_writable` and
|
||||||
|
by `delete`: a global document writes only its global key, a record held read-only is never replaced, and a key held as
|
||||||
|
another type's record or carried foreign is refused (`_refuse_retype`)."""
|
||||||
|
if document.global_document and key != GLOBAL_KEY:
|
||||||
|
raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}")
|
||||||
|
self._refuse_unread([key])
|
||||||
|
entry = document.catalog_entry()
|
||||||
|
probe = CultCacheEnvelope.create(key=key, type=document.type, payload=b"", schema_id=entry.schema_id, catalog_entry=entry)
|
||||||
|
return self._refuse_retype(document, [probe])
|
||||||
|
|
||||||
|
def _refuse_unread(self, keys: list[str]) -> None:
|
||||||
|
"""The one check every write and removal asks before any store is touched."""
|
||||||
|
for key in keys:
|
||||||
|
held = self._state.unread.get(key)
|
||||||
|
if held is None:
|
||||||
|
continue
|
||||||
|
type, slots = held
|
||||||
|
raise SchemaConflictError(
|
||||||
|
self._state.envelopes[type][key].schema_id or type,
|
||||||
|
[self._state.documents[type].catalog_entry().schema_name],
|
||||||
|
key,
|
||||||
|
message=(
|
||||||
|
f"{self._state.documents[type].catalog_entry().schema_name} record {key} is read-only: {slots.type_name} was read with "
|
||||||
|
f"{slots.slot_count} slots where this build declares {slots.declared_count} ({slots.occurrences} object(s) in the "
|
||||||
|
"record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or "
|
||||||
|
"write it from that build."
|
||||||
|
),
|
||||||
|
unread_slots=slots,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _foreign_refusal(record: ForeignRecord) -> SchemaConflictError:
|
||||||
|
return SchemaConflictError(
|
||||||
|
record.schema_id,
|
||||||
|
[record.schema_name],
|
||||||
|
record.key,
|
||||||
|
message=(
|
||||||
|
f"Record {record.key!r} is held by the store as a {record.schema_name} (schema id {record.schema_id!r}) that no "
|
||||||
|
"registered type claims; a write or removal there would replace what this cache cannot read. Register the type "
|
||||||
|
"that owns the schema id, or remove the record with the runtime that wrote it."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _retire_legacy_global(store: BackingStore, type: str, legacy_key: str) -> None:
|
def _retire_legacy_global(store: BackingStore, type: str, legacy_key: str) -> None:
|
||||||
"""The first write of a global loaded from a legacy key removes that record in the same write. If
|
"""The first write of a global loaded from a legacy key removes that record in the same write. If
|
||||||
|
|
@ -500,19 +727,29 @@ class CultCache:
|
||||||
|
|
||||||
def _delete(self, document: DocumentDefinition[Any], key: str) -> None:
|
def _delete(self, document: DocumentDefinition[Any], key: str) -> None:
|
||||||
self._assert_registered(document)
|
self._assert_registered(document)
|
||||||
|
# A key held as another type's record this document owns is removed, as a write of the key would replace it (C# Remove is by key).
|
||||||
|
owned = self._refuse_at(document, key)
|
||||||
store = self._store_for_type(document.type)
|
store = self._store_for_type(document.type)
|
||||||
legacy_key = self._state.legacy_global_keys.get(document.type) if key == GLOBAL_KEY else None
|
legacy_key = self._state.legacy_global_keys.get(document.type) if key == GLOBAL_KEY else None
|
||||||
if store is not None:
|
if store is not None:
|
||||||
store.delete(document.type, key if legacy_key is None else legacy_key)
|
# One record per key: the owned record when this document does not hold the key, else its own, in one store call.
|
||||||
|
holder = owned[0][0] if owned else document.type
|
||||||
|
store.delete(holder, key if legacy_key is None else legacy_key)
|
||||||
if legacy_key is not None:
|
if legacy_key is not None:
|
||||||
del self._state.legacy_global_keys[document.type]
|
del self._state.legacy_global_keys[document.type]
|
||||||
self._remove_lookups(document.type, key)
|
for other, _ in owned:
|
||||||
|
self._drop(other, key)
|
||||||
|
self._drop(document.type, key)
|
||||||
|
|
||||||
|
def _drop(self, type: str, key: str) -> None:
|
||||||
|
"""Removes the held record and its lookups from the cache's maps; the store is the caller's."""
|
||||||
|
self._remove_lookups(type, key)
|
||||||
for by_type in (self._state.values, self._state.envelopes, self._state.lookup_keys):
|
for by_type in (self._state.values, self._state.envelopes, self._state.lookup_keys):
|
||||||
held = by_type.get(document.type)
|
held = by_type.get(type)
|
||||||
if held is not None:
|
if held is not None:
|
||||||
held.pop(key, None)
|
held.pop(key, None)
|
||||||
if not held:
|
if not held:
|
||||||
by_type.pop(document.type, None)
|
by_type.pop(type, None)
|
||||||
|
|
||||||
@_mutating
|
@_mutating
|
||||||
def delete_global(self, document: DocumentDefinition[Any]) -> None:
|
def delete_global(self, document: DocumentDefinition[Any]) -> None:
|
||||||
|
|
@ -552,36 +789,76 @@ class CultCache:
|
||||||
name = None if name_extractor is None else extract_value(value, name_extractor)
|
name = None if name_extractor is None else extract_value(value, name_extractor)
|
||||||
indexes: dict[str, str] = {}
|
indexes: dict[str, str] = {}
|
||||||
for index, extractor in index_extractors.items():
|
for index, extractor in index_extractors.items():
|
||||||
index_value = extract_value(value, extractor)
|
index_value = index_text(extract_value(value, extractor))
|
||||||
if index_value is not None:
|
if index_value is not None:
|
||||||
indexes[index] = str(index_value)
|
indexes[index] = index_value
|
||||||
return _LookupKeys(None if name is None else str(name), indexes)
|
return _LookupKeys(None if name is None else str(name), indexes)
|
||||||
|
|
||||||
def _derive_lookups(
|
def _derive_lookups(
|
||||||
self,
|
self,
|
||||||
values_by_type: dict[str, dict[str, Any]],
|
records: list[tuple[str, str, Any]],
|
||||||
name_extractors: dict[str, str | Any],
|
name_extractors: dict[str, str | Any],
|
||||||
index_extractors: dict[str, dict[str, str | Any]],
|
index_extractors: dict[str, dict[str, str | Any]],
|
||||||
) -> tuple[_Names, _Indexes, _LookupKeysByType]:
|
) -> tuple[_Names, _Indexes, _LookupKeysByType]:
|
||||||
"""Builds name and index lookups for the given values in fresh dicts, touching no cache state."""
|
"""Builds name and index lookups for the given (type, key, value) records, judged whole and order-free, in fresh dicts, touching no cache state."""
|
||||||
names: _Names = {}
|
names: _Names = {}
|
||||||
indexes: _Indexes = {}
|
indexes: _Indexes = {}
|
||||||
lookup_keys: _LookupKeysByType = {}
|
lookup_keys: _LookupKeysByType = {}
|
||||||
for type, values in values_by_type.items():
|
landing: list[tuple[str, str, _LookupKeys]] = []
|
||||||
name_extractor = name_extractors.get(type)
|
for type, key, value in records:
|
||||||
type_index_extractors = index_extractors.get(type, {})
|
keys = self._lookup_keys_for(value, name_extractors.get(type), index_extractors.get(type, {}))
|
||||||
for key, value in values.items():
|
lookup_keys.setdefault(type, {})[key] = keys
|
||||||
keys = self._lookup_keys_for(value, name_extractor, type_index_extractors)
|
landing.append((type, key, keys))
|
||||||
lookup_keys.setdefault(type, {})[key] = keys
|
# Judged whole and order-free, over every type at once and on the file alone (never against what the cache held): a held value can
|
||||||
|
# have been mutated in place since it was admitted, and another process may have moved a value between records.
|
||||||
|
self._refuse_index_sharing(landing, {}, ())
|
||||||
|
for type in lookup_keys:
|
||||||
|
for key, keys in lookup_keys.get(type, {}).items():
|
||||||
if keys.name is not None:
|
if keys.name is not None:
|
||||||
names[(type, keys.name)] = key
|
names[(type, keys.name)] = key
|
||||||
for index, index_value in keys.indexes.items():
|
for index, index_value in keys.indexes.items():
|
||||||
indexes[(type, index, index_value)] = key
|
indexes[(type, index, index_value)] = key
|
||||||
return names, indexes, lookup_keys
|
return names, indexes, lookup_keys
|
||||||
|
|
||||||
|
def _refuse_index_sharing(
|
||||||
|
self,
|
||||||
|
landing: list[tuple[str, str, _LookupKeys]],
|
||||||
|
held: dict[tuple[str, str, str], str],
|
||||||
|
replaced: Iterable[tuple[str, str]],
|
||||||
|
) -> None:
|
||||||
|
"""The one index rule (variants:ruling:unique-indexes): within one type and one unique alias, no two records hold one
|
||||||
|
non-blank value, judged on the state the admission would leave and never on staging, file or pull order. The holders
|
||||||
|
of a value are the landing keys projecting it plus the held key unless this admission replaces it. Of every
|
||||||
|
(type, alias, value) with two or more distinct holders, the refusal names the least by (alias, its least key, its
|
||||||
|
second-least key, schema name), each compared by UTF-16 code unit, never the value."""
|
||||||
|
released = set(replaced)
|
||||||
|
landed: dict[tuple[str, str, str], list[str]] = {}
|
||||||
|
for type, key, keys in landing:
|
||||||
|
for index, index_value in keys.indexes.items():
|
||||||
|
landed.setdefault((type, index, index_value), []).append(key)
|
||||||
|
violations: list[tuple[bytes, bytes, bytes, bytes, str, str, str, str]] = []
|
||||||
|
for (type, index, index_value), keys in landed.items():
|
||||||
|
held_key = held.get((type, index, index_value))
|
||||||
|
if held_key is not None and (type, held_key) not in released:
|
||||||
|
keys = [*keys, held_key]
|
||||||
|
distinct = sorted(keys, key=_utf16_order)
|
||||||
|
if len(distinct) > 1:
|
||||||
|
document = self._state.documents.get(type)
|
||||||
|
schema_name = (document.schema_name if document is not None else None) or type
|
||||||
|
violations.append(
|
||||||
|
(_utf16_order(index), _utf16_order(distinct[0]), _utf16_order(distinct[1]), _utf16_order(schema_name),
|
||||||
|
index, distinct[0], distinct[1], schema_name)
|
||||||
|
)
|
||||||
|
if violations:
|
||||||
|
*_, index, first, second, schema_name = min(violations, key=lambda violation: violation[:4])
|
||||||
|
raise CultCacheError(
|
||||||
|
f"Records {first} and {second} ({schema_name}) both hold unique index '{index}'; "
|
||||||
|
"an index value names one record."
|
||||||
|
)
|
||||||
|
|
||||||
def _remove_lookups(self, type: str, key: str) -> None:
|
def _remove_lookups(self, type: str, key: str) -> None:
|
||||||
"""Removes the lookups the held record produced at admission, handing a name or index value to
|
"""Removes the lookups the held record produced at admission, handing a name to another held
|
||||||
another held record that produced it too. Reads stored lookup keys only; runs no extractor."""
|
record that produced it too (an index value has one holder). Reads stored lookup keys only; runs no extractor."""
|
||||||
held = self._state.lookup_keys.get(type, {})
|
held = self._state.lookup_keys.get(type, {})
|
||||||
keys = held.get(key)
|
keys = held.get(key)
|
||||||
if keys is None:
|
if keys is None:
|
||||||
|
|
@ -596,9 +873,6 @@ class CultCache:
|
||||||
if self._state.indexes.get(lookup_key) != key:
|
if self._state.indexes.get(lookup_key) != key:
|
||||||
continue
|
continue
|
||||||
del self._state.indexes[lookup_key]
|
del self._state.indexes[lookup_key]
|
||||||
for candidate_key, candidate in held.items():
|
|
||||||
if candidate_key != key and candidate.indexes.get(index) == index_value:
|
|
||||||
self._state.indexes[lookup_key] = candidate_key
|
|
||||||
|
|
||||||
def _assert_registered(self, document: DocumentDefinition[Any]) -> None:
|
def _assert_registered(self, document: DocumentDefinition[Any]) -> None:
|
||||||
if self._state.documents.get(document.type) is not document:
|
if self._state.documents.get(document.type) is not document:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import threading
|
||||||
from dataclasses import MISSING, dataclass, field
|
from dataclasses import MISSING, dataclass, field
|
||||||
from collections.abc import Mapping, Sequence, Set
|
from collections.abc import Mapping, Sequence, Set
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
|
|
@ -30,6 +31,49 @@ MAX_SLOT_KEY = 65535
|
||||||
_STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,)
|
_STACK_FAULTS: tuple[type[BaseException], ...] = (_MsgpackStackError,)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UnreadSlots:
|
||||||
|
"""What a payload decode found past the slots this runtime declares, at any depth: the first type read with extra slots, how
|
||||||
|
many slots it carried against how many it declares, and how many objects in the payload did. It holds no slot value, so a
|
||||||
|
refusal can name it. `depth0_only` is True when every mark sat in the document's own record (the half of consent the payload
|
||||||
|
decides). A record carrying it is read-only unless the schema comparison explains every mark (see `CultCache.is_writable`)."""
|
||||||
|
|
||||||
|
type_name: str
|
||||||
|
slot_count: int
|
||||||
|
declared_count: int
|
||||||
|
occurrences: int
|
||||||
|
depth0_only: bool
|
||||||
|
|
||||||
|
|
||||||
|
class _SlotScope:
|
||||||
|
"""Opened by `DocumentDefinition.decode_payload_marked`; the record codec and the union codec note into it. It is thread-local,
|
||||||
|
so no codec signature carries it, and a codec decoded outside a payload decode notes nothing."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.depth = 0 # records entered: 0 while the document's own record is read
|
||||||
|
self.first: tuple[str, int, int] | None = None
|
||||||
|
self.occurrences = 0
|
||||||
|
self.depth0_only = True
|
||||||
|
|
||||||
|
def note(self, type_name: str, slot_count: int, declared_count: int, depth0: bool) -> None:
|
||||||
|
if self.first is None:
|
||||||
|
self.first = (type_name, slot_count, declared_count)
|
||||||
|
self.occurrences += 1
|
||||||
|
self.depth0_only = self.depth0_only and depth0
|
||||||
|
|
||||||
|
def result(self) -> UnreadSlots | None:
|
||||||
|
if self.first is None:
|
||||||
|
return None
|
||||||
|
return UnreadSlots(*self.first, self.occurrences, self.depth0_only)
|
||||||
|
|
||||||
|
|
||||||
|
_scope = threading.local()
|
||||||
|
|
||||||
|
|
||||||
|
def _current_scope() -> _SlotScope | None:
|
||||||
|
return getattr(_scope, "open", None)
|
||||||
|
|
||||||
|
|
||||||
_CONTAINERS = (Mapping, Sequence, Set)
|
_CONTAINERS = (Mapping, Sequence, Set)
|
||||||
_SCALAR_SEQUENCES = (str, bytes, bytearray, memoryview)
|
_SCALAR_SEQUENCES = (str, bytes, bytearray, memoryview)
|
||||||
|
|
||||||
|
|
@ -157,6 +201,11 @@ class DocumentDefinition(Generic[T]):
|
||||||
global_document: bool = False
|
global_document: bool = False
|
||||||
name: str | Extractor[T] | None = None
|
name: str | Extractor[T] | None = None
|
||||||
indexes: dict[str, str | Extractor[T]] = field(default_factory=dict)
|
indexes: dict[str, str | Extractor[T]] = field(default_factory=dict)
|
||||||
|
"""Unique lookups (C# `[CultIndex]`): no two records of the type hold one non-blank value, judged at every commit and
|
||||||
|
load. `get_by_index` answers the one holder."""
|
||||||
|
roles: dict[str, str | Extractor[T]] = field(default_factory=dict)
|
||||||
|
"""Selection aliases (C# `[CultRole]`): many records may share a value. Never indexed for lookup, never refuse; the
|
||||||
|
member's catalog `index_alias` is the same either way."""
|
||||||
payload_encoder: Callable[[Any], bytes] = _json_payload_encoder
|
payload_encoder: Callable[[Any], bytes] = _json_payload_encoder
|
||||||
payload_decoder: Callable[[bytes], Any] = _json_payload_decoder
|
payload_decoder: Callable[[bytes], Any] = _json_payload_decoder
|
||||||
schema_id: str | None = None
|
schema_id: str | None = None
|
||||||
|
|
@ -180,15 +229,33 @@ class DocumentDefinition(Generic[T]):
|
||||||
return self.payload_encoder(tree)
|
return self.payload_encoder(tree)
|
||||||
|
|
||||||
def decode_payload(self, payload: bytes) -> T:
|
def decode_payload(self, payload: bytes) -> T:
|
||||||
|
return self.decode_payload_marked(payload)[0]
|
||||||
|
|
||||||
|
def decode_payload_marked(self, payload: bytes) -> tuple[T, UnreadSlots | None]:
|
||||||
|
"""The value and what the decode found past the declared slots, at any depth. The payload depth check runs first: a
|
||||||
|
payload past the limit is refused, never marked."""
|
||||||
try:
|
try:
|
||||||
tree = self.payload_decoder(payload)
|
tree = self.payload_decoder(payload)
|
||||||
except _STACK_FAULTS:
|
except _STACK_FAULTS:
|
||||||
raise CultCachePayloadDepthError(self.type, "read") from None
|
raise CultCachePayloadDepthError(self.type, "read") from None
|
||||||
_refuse_past_depth_limit(tree, self.type, "read")
|
_refuse_past_depth_limit(tree, self.type, "read")
|
||||||
value = self.decode(tree)
|
outer, opened = _current_scope(), _SlotScope()
|
||||||
|
_scope.open = opened
|
||||||
|
try:
|
||||||
|
value = self.decode(tree)
|
||||||
|
finally:
|
||||||
|
_scope.open = outer
|
||||||
if self.validate:
|
if self.validate:
|
||||||
self.validate(value)
|
self.validate(value)
|
||||||
return value
|
return value, opened.result()
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
# Judged here so every constructor passes it: the factories, the dataclass constructor and dataclasses.replace.
|
||||||
|
for alias in self.roles:
|
||||||
|
if alias in self.indexes:
|
||||||
|
raise ValueError(
|
||||||
|
f"Document type {self.type} declares '{alias}' as both an index and a role; an index already names its role."
|
||||||
|
)
|
||||||
|
|
||||||
def catalog_entry(self) -> CultCacheSchemaCatalogEntry:
|
def catalog_entry(self) -> CultCacheSchemaCatalogEntry:
|
||||||
if self._catalog_entry is not None:
|
if self._catalog_entry is not None:
|
||||||
|
|
@ -241,6 +308,7 @@ def define_document_type(
|
||||||
global_document: bool = False,
|
global_document: bool = False,
|
||||||
name: str | Extractor[T] | None = None,
|
name: str | Extractor[T] | None = None,
|
||||||
indexes: dict[str, str | Extractor[T]] | None = None,
|
indexes: dict[str, str | Extractor[T]] | None = None,
|
||||||
|
roles: dict[str, str | Extractor[T]] | None = None,
|
||||||
payload_encoder: Callable[[Any], bytes] = _json_payload_encoder,
|
payload_encoder: Callable[[Any], bytes] = _json_payload_encoder,
|
||||||
payload_decoder: Callable[[bytes], Any] = _json_payload_decoder,
|
payload_decoder: Callable[[bytes], Any] = _json_payload_decoder,
|
||||||
schema_id: str | None = None,
|
schema_id: str | None = None,
|
||||||
|
|
@ -259,6 +327,7 @@ def define_document_type(
|
||||||
global_document=global_document,
|
global_document=global_document,
|
||||||
name=name,
|
name=name,
|
||||||
indexes=indexes or {},
|
indexes=indexes or {},
|
||||||
|
roles=roles or {},
|
||||||
payload_encoder=payload_encoder,
|
payload_encoder=payload_encoder,
|
||||||
payload_decoder=payload_decoder,
|
payload_decoder=payload_decoder,
|
||||||
schema_id=schema_id,
|
schema_id=schema_id,
|
||||||
|
|
@ -280,6 +349,7 @@ def define_database_entry_type(
|
||||||
global_document: bool = False,
|
global_document: bool = False,
|
||||||
name: str | Extractor[T] | None = None,
|
name: str | Extractor[T] | None = None,
|
||||||
indexes: dict[str, str | Extractor[T]] | None = None,
|
indexes: dict[str, str | Extractor[T]] | None = None,
|
||||||
|
roles: dict[str, str | Extractor[T]] | None = None,
|
||||||
schema_id: str | None = None,
|
schema_id: str | None = None,
|
||||||
schema_name: str | None = None,
|
schema_name: str | None = None,
|
||||||
schema_version: str | None = None,
|
schema_version: str | None = None,
|
||||||
|
|
@ -296,7 +366,7 @@ def define_database_entry_type(
|
||||||
|
|
||||||
normalized = _normalize_fields(type, fields)
|
normalized = _normalize_fields(type, fields)
|
||||||
member_name_lookup = name if isinstance(name, str) else None
|
member_name_lookup = name if isinstance(name, str) else None
|
||||||
index_lookup = indexes or {}
|
index_lookup = {**(roles or {}), **(indexes or {})}
|
||||||
|
|
||||||
def members() -> tuple[CultCacheSchemaCatalogMember, ...]:
|
def members() -> tuple[CultCacheSchemaCatalogMember, ...]:
|
||||||
return tuple(
|
return tuple(
|
||||||
|
|
@ -323,6 +393,7 @@ def define_database_entry_type(
|
||||||
global_document=global_document,
|
global_document=global_document,
|
||||||
name=name,
|
name=name,
|
||||||
indexes=indexes,
|
indexes=indexes,
|
||||||
|
roles=roles,
|
||||||
payload_encoder=_pack_tree,
|
payload_encoder=_pack_tree,
|
||||||
payload_decoder=_messagepack_payload_decoder,
|
payload_decoder=_messagepack_payload_decoder,
|
||||||
schema_id=schema_id,
|
schema_id=schema_id,
|
||||||
|
|
@ -401,6 +472,9 @@ def define_union(
|
||||||
raise ValueError(f"Union {name!r} value is malformed: expected [armKey, [armSlots]]")
|
raise ValueError(f"Union {name!r} value is malformed: expected [armKey, [armSlots]]")
|
||||||
arm_name, codec = by_key[key]
|
arm_name, codec = by_key[key]
|
||||||
if codec is None:
|
if codec is None:
|
||||||
|
scope = _current_scope()
|
||||||
|
if scope is not None and raw[1]:
|
||||||
|
scope.note(arm_name, len(raw[1]), 0, False)
|
||||||
return UnionArm(arm_name, None)
|
return UnionArm(arm_name, None)
|
||||||
try:
|
try:
|
||||||
return UnionArm(arm_name, codec.decode(raw[1]))
|
return UnionArm(arm_name, codec.decode(raw[1]))
|
||||||
|
|
@ -462,17 +536,26 @@ def _record_codec(
|
||||||
def decode(raw: Any) -> Any:
|
def decode(raw: Any) -> Any:
|
||||||
if not isinstance(raw, list):
|
if not isinstance(raw, list):
|
||||||
raise ValueError(f"DatabaseEntry document type {type!r} expected a MessagePack array")
|
raise ValueError(f"DatabaseEntry document type {type!r} expected a MessagePack array")
|
||||||
|
scope = _current_scope()
|
||||||
|
if scope is not None and len(raw) > max_key + 1:
|
||||||
|
scope.note(type, len(raw), max_key + 1, scope.depth == 0)
|
||||||
values: dict[str, Any] = {}
|
values: dict[str, Any] = {}
|
||||||
for field in normalized:
|
if scope is not None:
|
||||||
if field.key < len(raw):
|
scope.depth += 1
|
||||||
try:
|
try:
|
||||||
values[field.name] = field.codec.decode(raw[field.key])
|
for field in normalized:
|
||||||
except ValueError as error:
|
if field.key < len(raw):
|
||||||
raise ValueError(f"{type} slot {field.key} ({field.name}): {error}") from None
|
try:
|
||||||
elif field.default is MISSING:
|
values[field.name] = field.codec.decode(raw[field.key])
|
||||||
values[field.name] = None
|
except ValueError as error:
|
||||||
else:
|
raise ValueError(f"{type} slot {field.key} ({field.name}): {error}") from None
|
||||||
values[field.name] = field.default() if callable(field.default) else field.default
|
elif field.default is MISSING:
|
||||||
|
values[field.name] = None
|
||||||
|
else:
|
||||||
|
values[field.name] = field.default() if callable(field.default) else field.default
|
||||||
|
finally:
|
||||||
|
if scope is not None:
|
||||||
|
scope.depth -= 1
|
||||||
if cls is None:
|
if cls is None:
|
||||||
return values
|
return values
|
||||||
return cls(**values)
|
return cls(**values)
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ from typing import Any, Callable, Iterator, TypeVar
|
||||||
|
|
||||||
import msgpack
|
import msgpack
|
||||||
|
|
||||||
|
from .documents import UnreadSlots
|
||||||
from .backing_store import (
|
from .backing_store import (
|
||||||
CultCacheEnvelope,
|
CultCacheEnvelope,
|
||||||
CultCacheSchemaCatalogEntry,
|
CultCacheSchemaCatalogEntry,
|
||||||
|
|
@ -46,9 +47,19 @@ class SchemaConflictError(ValueError):
|
||||||
"""A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id
|
"""A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id
|
||||||
and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names
|
and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names
|
||||||
involved (`schema_names`) and a record key (`record_key`). A cache raises it too, with an empty record key, when a
|
involved (`schema_names`) and a record key (`record_key`). A cache raises it too, with an empty record key, when a
|
||||||
document registers a schema id another registered document already carries."""
|
document registers a schema id another registered document already carries. `unread_slots` is None except for the refusal to
|
||||||
|
write or remove a record held read-only because its payload carries slots past the declared ones; its message names the type
|
||||||
|
and both slot counts and never holds a slot value."""
|
||||||
|
|
||||||
def __init__(self, schema_id: str, schema_names: list[str], record_key: str, *, message: str | None = None) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
schema_id: str,
|
||||||
|
schema_names: list[str],
|
||||||
|
record_key: str,
|
||||||
|
*,
|
||||||
|
message: str | None = None,
|
||||||
|
unread_slots: UnreadSlots | None = None,
|
||||||
|
) -> None:
|
||||||
super().__init__(
|
super().__init__(
|
||||||
message
|
message
|
||||||
or f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}"
|
or f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}"
|
||||||
|
|
@ -56,6 +67,7 @@ class SchemaConflictError(ValueError):
|
||||||
self.schema_id = schema_id
|
self.schema_id = schema_id
|
||||||
self.schema_names = schema_names
|
self.schema_names = schema_names
|
||||||
self.record_key = record_key
|
self.record_key = record_key
|
||||||
|
self.unread_slots = unread_slots
|
||||||
|
|
||||||
|
|
||||||
class JsonLinesBackingStore:
|
class JsonLinesBackingStore:
|
||||||
|
|
@ -245,11 +257,44 @@ def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultC
|
||||||
return [entry for entry, _ in chosen.values()]
|
return [entry for entry, _ in chosen.values()]
|
||||||
|
|
||||||
|
|
||||||
|
def retyped(held: CultCacheEnvelope, written: CultCacheEnvelope) -> SchemaConflictError:
|
||||||
|
"""The refusal of a write that would put a second type under a key the store already holds (C# CacheBackingStore.Retyped)."""
|
||||||
|
return SchemaConflictError(
|
||||||
|
_schema_id_for(held),
|
||||||
|
[held.type],
|
||||||
|
held.key,
|
||||||
|
message=(
|
||||||
|
f"Record '{held.key}' is a {held.type} (schema id '{_schema_id_for(held)}'); writing a {written.type} there would "
|
||||||
|
"retype the key. Remove the record first, in the same batch or an earlier commit, then add the other type's."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def written_owns(held: CultCacheEnvelope, written: CultCacheEnvelope) -> bool:
|
||||||
|
"""Whether a write of `written` may replace `held` at its key (C# WrittenOwns): the written type's entry declares the held
|
||||||
|
record's schema id (its own id or one it lists compatible), or it is the type of the same schema name. The same-name half
|
||||||
|
stands in for C#'s "reads the published entry whole": this runtime does not compare shapes, so it never refuses a
|
||||||
|
same-name type for dropping slots."""
|
||||||
|
stored_id = _schema_id_for(held)
|
||||||
|
entry = written.catalog_entry if written.catalog_entry is not None else _default_catalog_entry(written)
|
||||||
|
return stored_id == entry.schema_id or stored_id in entry.compatible_schema_ids or held.type == written.type
|
||||||
|
|
||||||
|
|
||||||
def _encode_snapshot(
|
def _encode_snapshot(
|
||||||
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
|
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
|
||||||
) -> list[Any]:
|
) -> list[Any]:
|
||||||
replaced = {(envelope.type, envelope.key) for envelope in supplied}
|
# A key names one record whatever its type, so a supplied record replaces the arrived one by key alone, and only of the
|
||||||
kept = [envelope for envelope in arrived if (envelope.type, envelope.key) not in replaced]
|
# type that owns it: retyping a key is a removal and an add, so a record another type owns under the key refuses the write.
|
||||||
|
held_by_key = {envelope.key: envelope for envelope in arrived}
|
||||||
|
landing: dict[str, CultCacheEnvelope] = {}
|
||||||
|
for envelope in supplied:
|
||||||
|
for held in (landing.get(envelope.key), held_by_key.get(envelope.key)):
|
||||||
|
if held is not None and not written_owns(held, envelope):
|
||||||
|
raise retyped(held, envelope)
|
||||||
|
landing[envelope.key] = envelope
|
||||||
|
supplied = list(landing.values())
|
||||||
|
replaced = set(landing)
|
||||||
|
kept = [envelope for envelope in arrived if envelope.key not in replaced]
|
||||||
envelopes = [*kept, *supplied]
|
envelopes = [*kept, *supplied]
|
||||||
catalog = [
|
catalog = [
|
||||||
_encode_catalog_entry(entry)
|
_encode_catalog_entry(entry)
|
||||||
|
|
@ -400,6 +445,7 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None
|
||||||
catalog_by_schema_id.setdefault(compatible_schema_id, entry)
|
catalog_by_schema_id.setdefault(compatible_schema_id, entry)
|
||||||
|
|
||||||
envelopes: list[CultCacheEnvelope] = []
|
envelopes: list[CultCacheEnvelope] = []
|
||||||
|
seen: set[str] = set()
|
||||||
for raw_record in decoded[2]:
|
for raw_record in decoded[2]:
|
||||||
if not isinstance(raw_record, list) or len(raw_record) < 4:
|
if not isinstance(raw_record, list) or len(raw_record) < 4:
|
||||||
raise ValueError("CultCache persisted records must be MessagePack arrays")
|
raise ValueError("CultCache persisted records must be MessagePack arrays")
|
||||||
|
|
@ -412,6 +458,9 @@ def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None
|
||||||
key, schema_id, stored_at, payload = raw_record
|
key, schema_id, stored_at, payload = raw_record
|
||||||
if not isinstance(key, str) or not key:
|
if not isinstance(key, str) or not key:
|
||||||
raise ValueError("CultCache persisted records must declare a key")
|
raise ValueError("CultCache persisted records must declare a key")
|
||||||
|
if key in seen:
|
||||||
|
raise ValueError(f"CultCache record {key!r} appears more than once")
|
||||||
|
seen.add(key)
|
||||||
if not isinstance(schema_id, str) or not schema_id:
|
if not isinstance(schema_id, str) or not schema_id:
|
||||||
raise ValueError("CultCache persisted records must declare a schema id")
|
raise ValueError("CultCache persisted records must declare a schema id")
|
||||||
if not isinstance(stored_at, str) or not stored_at:
|
if not isinstance(stored_at, str) or not stored_at:
|
||||||
|
|
@ -437,9 +486,13 @@ def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]:
|
||||||
if not isinstance(decoded, list):
|
if not isinstance(decoded, list):
|
||||||
raise ValueError("CultCache MessagePack store is not a recognized snapshot")
|
raise ValueError("CultCache MessagePack store is not a recognized snapshot")
|
||||||
envelopes: list[CultCacheEnvelope] = []
|
envelopes: list[CultCacheEnvelope] = []
|
||||||
|
seen: set[Any] = set()
|
||||||
for item in decoded:
|
for item in decoded:
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
raise ValueError("CultCache legacy envelopes must be MessagePack maps")
|
raise ValueError("CultCache legacy envelopes must be MessagePack maps")
|
||||||
|
if item.get("key") in seen:
|
||||||
|
raise ValueError(f"CultCache record {item['key']!r} appears more than once")
|
||||||
|
seen.add(item.get("key"))
|
||||||
try:
|
try:
|
||||||
payload = item["payload"]
|
payload = item["payload"]
|
||||||
envelopes.append(
|
envelopes.append(
|
||||||
|
|
@ -484,12 +537,14 @@ def _decode_catalog_entry(value: Any) -> CultCacheSchemaCatalogEntry:
|
||||||
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid compatible schema ids')
|
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid compatible schema ids')
|
||||||
if not isinstance(members, list):
|
if not isinstance(members, list):
|
||||||
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid members')
|
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid members')
|
||||||
|
if not isinstance(canonical_schema_json, str):
|
||||||
|
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has an invalid canonical schema')
|
||||||
return CultCacheSchemaCatalogEntry(
|
return CultCacheSchemaCatalogEntry(
|
||||||
schema_id=schema_id,
|
schema_id=schema_id,
|
||||||
schema_name=schema_name,
|
schema_name=schema_name,
|
||||||
schema_version=schema_version if isinstance(schema_version, str) and schema_version else f"{schema_name}.v1",
|
schema_version=schema_version if isinstance(schema_version, str) and schema_version else f"{schema_name}.v1",
|
||||||
content_hash=content_hash if isinstance(content_hash, str) and content_hash else schema_id,
|
content_hash=content_hash if isinstance(content_hash, str) and content_hash else schema_id,
|
||||||
canonical_schema_json=canonical_schema_json if isinstance(canonical_schema_json, str) else "",
|
canonical_schema_json=canonical_schema_json,
|
||||||
compatible_schema_ids=tuple(compatible_schema_ids),
|
compatible_schema_ids=tuple(compatible_schema_ids),
|
||||||
members=tuple(_decode_catalog_member(member) for member in members),
|
members=tuple(_decode_catalog_member(member) for member in members),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -103,20 +103,20 @@ class CultCacheTests(unittest.TestCase):
|
||||||
self.assertEqual(cache.get_key_by_name(document, "Elixir"), "item:potion")
|
self.assertEqual(cache.get_key_by_name(document, "Elixir"), "item:potion")
|
||||||
self.assertEqual(cache.get_by_index(document, "category", "Rare").name, "Elixir")
|
self.assertEqual(cache.get_by_index(document, "category", "Rare").name, "Elixir")
|
||||||
|
|
||||||
def test_incremental_lookup_delete_restores_duplicate_owner(self) -> None:
|
def test_incremental_lookup_delete_restores_duplicate_name_owner(self) -> None:
|
||||||
document = item_doc()
|
document = item_doc()
|
||||||
cache = CultCache()
|
cache = CultCache()
|
||||||
cache.register_document_type(document)
|
cache.register_document_type(document)
|
||||||
|
|
||||||
cache.put(document, "item:first", Item(name="Potion", category="Consumable", value=50))
|
cache.put(document, "item:first", Item(name="Potion", category="Consumable", value=50))
|
||||||
cache.put(document, "item:second", Item(name="Potion", category="Consumable", value=60))
|
cache.put(document, "item:second", Item(name="Potion", category="Rare", value=60))
|
||||||
self.assertEqual(cache.get_key_by_name(document, "Potion"), "item:second")
|
self.assertEqual(cache.get_key_by_name(document, "Potion"), "item:second")
|
||||||
self.assertEqual(cache.get_key_by_index(document, "category", "Consumable"), "item:second")
|
|
||||||
|
|
||||||
cache.delete(document, "item:second")
|
cache.delete(document, "item:second")
|
||||||
|
|
||||||
self.assertEqual(cache.get_key_by_name(document, "Potion"), "item:first")
|
self.assertEqual(cache.get_key_by_name(document, "Potion"), "item:first")
|
||||||
self.assertEqual(cache.get_key_by_index(document, "category", "Consumable"), "item:first")
|
self.assertEqual(cache.get_key_by_index(document, "category", "Consumable"), "item:first")
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "category", "Rare"))
|
||||||
|
|
||||||
def test_database_entry_formatter_uses_slot_indexed_messagepack_array(self) -> None:
|
def test_database_entry_formatter_uses_slot_indexed_messagepack_array(self) -> None:
|
||||||
try:
|
try:
|
||||||
|
|
@ -1567,15 +1567,19 @@ class CultCacheTests(unittest.TestCase):
|
||||||
# Every vector in the folder has a manifest row.
|
# Every vector in the folder has a manifest row.
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sorted(path.name for path in root.glob("*.bin")),
|
sorted(path.name for path in root.glob("*.bin")),
|
||||||
sorted(vector for vector, *_ in rows if not vector.startswith("..")),
|
sorted(vector for vector, *_ in rows if not vector.startswith(("..", "@"))),
|
||||||
)
|
)
|
||||||
envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all()
|
envelopes = SingleFileMessagePackBackingStore(self._C2A_VECTORS / "v3-base.msgpack").pull_all()
|
||||||
for vector, _, _, _, python in rows:
|
for vector, _, _, _, python in rows:
|
||||||
content = (root / vector).read_bytes()
|
# "@directory": the walker makes an empty directory at the store path; it is refused and left untouched.
|
||||||
|
content = None if vector == "@directory" else (root / vector).read_bytes()
|
||||||
for operation in ("pull_all", "push_all", "push", "delete"):
|
for operation in ("pull_all", "push_all", "push", "delete"):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
store_path = Path(tmp) / "store.msgpack"
|
store_path = Path(tmp) / "store.msgpack"
|
||||||
store_path.write_bytes(content)
|
if content is None:
|
||||||
|
store_path.mkdir()
|
||||||
|
else:
|
||||||
|
store_path.write_bytes(content)
|
||||||
store = SingleFileMessagePackBackingStore(store_path)
|
store = SingleFileMessagePackBackingStore(store_path)
|
||||||
run = {
|
run = {
|
||||||
"pull_all": store.pull_all,
|
"pull_all": store.pull_all,
|
||||||
|
|
@ -1598,12 +1602,129 @@ class CultCacheTests(unittest.TestCase):
|
||||||
self.assertIn(envelopes[0].key, keys, f"{vector} {operation}")
|
self.assertIn(envelopes[0].key, keys, f"{vector} {operation}")
|
||||||
else:
|
else:
|
||||||
self.assertNotIn(envelopes[0].key, keys, f"{vector} {operation}")
|
self.assertNotIn(envelopes[0].key, keys, f"{vector} {operation}")
|
||||||
|
elif content is None:
|
||||||
|
# A path that cannot be reached as a file stays an OSError (_read_store): not a store that failed to decode.
|
||||||
|
with self.assertRaises(OSError, msg=f"{vector} {operation}"):
|
||||||
|
run()
|
||||||
|
self.assertTrue(store_path.is_dir() and not any(store_path.iterdir()), f"{vector} {operation} touched the directory")
|
||||||
else:
|
else:
|
||||||
with self.assertRaises(StoreUnreadableError, msg=f"{vector} {operation}") as refused:
|
with self.assertRaises(StoreUnreadableError, msg=f"{vector} {operation}") as refused:
|
||||||
run()
|
run()
|
||||||
self.assertIsNotNone(refused.exception.__cause__, f"{vector} {operation}")
|
self.assertIsNotNone(refused.exception.__cause__, f"{vector} {operation}")
|
||||||
self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read")
|
self.assertEqual(store_path.read_bytes(), content, f"{vector} {operation} rewrote a file it cannot read")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _rec(type: str, key: str, second: int = 0) -> CultCacheEnvelope:
|
||||||
|
return CultCacheEnvelope(key=key, type=type, payload=b"\x90", stored_at=f"2026-09-30T00:00:0{second}Z")
|
||||||
|
|
||||||
|
def test_a_supplied_record_replaces_an_arrived_one_of_the_same_type_by_key(self) -> None:
|
||||||
|
import msgpack # type: ignore
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.msgpack"
|
||||||
|
store = SingleFileMessagePackBackingStore(path)
|
||||||
|
store.push(self._rec("first", "k"))
|
||||||
|
store.push(self._rec("first", "k", 1))
|
||||||
|
store.push_all([self._rec("first", "k", 2), self._rec("first", "k", 3)])
|
||||||
|
records = msgpack.unpackb(path.read_bytes(), raw=False)[2]
|
||||||
|
self.assertEqual([record[0] for record in records], ["k"])
|
||||||
|
self.assertEqual([(e.type, e.stored_at) for e in store.pull_all()], [("first", "2026-09-30T00:00:03Z")])
|
||||||
|
|
||||||
|
def test_a_write_that_would_retype_a_key_is_refused_and_leaves_the_file(self) -> None:
|
||||||
|
from cultcache_py import SchemaConflictError
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.msgpack"
|
||||||
|
store = SingleFileMessagePackBackingStore(path)
|
||||||
|
store.push(self._rec("first", "k"))
|
||||||
|
content = path.read_bytes()
|
||||||
|
for run in (
|
||||||
|
lambda: store.push(self._rec("second", "k", 1)),
|
||||||
|
lambda: store.push_all([self._rec("other", "j"), self._rec("second", "k", 1)]),
|
||||||
|
):
|
||||||
|
with self.assertRaises(SchemaConflictError) as refused:
|
||||||
|
run()
|
||||||
|
self.assertEqual(refused.exception.record_key, "k")
|
||||||
|
self.assertEqual(refused.exception.schema_names, ["first"])
|
||||||
|
self.assertIn("would retype the key", str(refused.exception))
|
||||||
|
self.assertEqual(path.read_bytes(), content)
|
||||||
|
# Retyping is a removal and an add.
|
||||||
|
store.delete("first", "k")
|
||||||
|
store.push(self._rec("second", "k", 1))
|
||||||
|
self.assertEqual([e.type for e in SingleFileMessagePackBackingStore(path).pull_all()], ["second"])
|
||||||
|
|
||||||
|
def test_a_retype_is_judged_by_the_stored_schema_id_the_written_type_declares_not_by_name(self) -> None:
|
||||||
|
from cultcache_py import SchemaConflictError
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.msgpack"
|
||||||
|
store = SingleFileMessagePackBackingStore(path)
|
||||||
|
store.push(self._writer_record("k", "old", "id-old", self._writer_entry("id-old", "old", "h1", ("id-old",))))
|
||||||
|
content = path.read_bytes()
|
||||||
|
# A renamed type that lists the stored id compatible owns the record, so its write replaces it.
|
||||||
|
renamed = self._writer_record("k", "new", "id-new", self._writer_entry("id-new", "new", "h2", ("id-new", "id-old")))
|
||||||
|
# A type that lists nothing of the stored id and has another name does not.
|
||||||
|
stranger = self._writer_record("k", "stranger", "id-x", self._writer_entry("id-x", "stranger", "h3", ("id-x",)))
|
||||||
|
for run in (lambda: store.push(stranger), lambda: store.push_all([stranger])):
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
run()
|
||||||
|
self.assertEqual(path.read_bytes(), content)
|
||||||
|
store.push(renamed)
|
||||||
|
self.assertEqual([(e.type, e.schema_id) for e in SingleFileMessagePackBackingStore(path).pull_all()], [("new", "id-new")])
|
||||||
|
|
||||||
|
def test_a_batch_naming_one_key_under_two_types_is_refused_whole(self) -> None:
|
||||||
|
from cultcache_py import SchemaConflictError
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.msgpack"
|
||||||
|
store = SingleFileMessagePackBackingStore(path)
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
store.push_all([self._rec("first", "k"), self._rec("second", "k", 1)])
|
||||||
|
self.assertFalse(path.exists())
|
||||||
|
|
||||||
|
def test_a_cache_put_of_a_second_type_under_a_key_is_refused_and_persists_nothing_new(self) -> None:
|
||||||
|
from cultcache_py import SchemaConflictError
|
||||||
|
|
||||||
|
other = define_document_type("other", encode=lambda item: asdict(item), decode=lambda raw: Item(**raw))
|
||||||
|
document = item_doc()
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.msgpack"
|
||||||
|
|
||||||
|
def build() -> CultCache:
|
||||||
|
cache = (
|
||||||
|
CultCache.builder()
|
||||||
|
.register_document_type(document)
|
||||||
|
.register_document_type(other)
|
||||||
|
.add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
return cache
|
||||||
|
|
||||||
|
cache = build()
|
||||||
|
cache.put(document, "shared", Item(name="A", category="c", value=1))
|
||||||
|
content = path.read_bytes()
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.put(other, "shared", Item(name="B", category="c", value=2))
|
||||||
|
self.assertEqual(path.read_bytes(), content)
|
||||||
|
self.assertIsNone(cache.get(other, "shared"))
|
||||||
|
reloaded = build()
|
||||||
|
self.assertEqual(reloaded.get_required(document, "shared").value, 1)
|
||||||
|
self.assertIsNone(reloaded.get(other, "shared"))
|
||||||
|
cache.delete(document, "shared")
|
||||||
|
cache.put(other, "shared", Item(name="B", category="c", value=2))
|
||||||
|
self.assertEqual(build().get_required(other, "shared").value, 2)
|
||||||
|
|
||||||
|
def test_a_catalog_entry_with_a_non_string_canonical_schema_is_unreadable(self) -> None:
|
||||||
|
# Direct probe of the c0 vector; its manifest row is added by the TS cut.
|
||||||
|
vector = self._C2A_VECTORS.parent / "document-variants-c0" / "current-catalog-bad-canonical.msgpack"
|
||||||
|
content = vector.read_bytes()
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.msgpack"
|
||||||
|
path.write_bytes(content)
|
||||||
|
with self.assertRaises(StoreUnreadableError):
|
||||||
|
SingleFileMessagePackBackingStore(path).pull_all()
|
||||||
|
self.assertEqual(path.read_bytes(), content)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,6 @@ item_doc = define_document_type(
|
||||||
encode=lambda item: asdict(item),
|
encode=lambda item: asdict(item),
|
||||||
decode=lambda raw: Item(**raw),
|
decode=lambda raw: Item(**raw),
|
||||||
name="name",
|
name="name",
|
||||||
indexes={"category": "category"},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
316
packages/cultcache-py/tests/test_key_identity_vector.py
Normal file
316
packages/cultcache-py/tests/test_key_identity_vector.py
Normal file
|
|
@ -0,0 +1,316 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
|
||||||
|
from cultcache_py.cache import CultCacheError
|
||||||
|
from cultcache_py import (
|
||||||
|
CultCache,
|
||||||
|
|
||||||
|
JsonLinesBackingStore,
|
||||||
|
SchemaConflictError,
|
||||||
|
SingleFileMessagePackBackingStore,
|
||||||
|
database_entry_field,
|
||||||
|
define_database_entry_type,
|
||||||
|
leaves as t,
|
||||||
|
)
|
||||||
|
|
||||||
|
# variants:ruling:cache-refuses-retype, write-ownership-by-written-type, unheld-removal-refused-at-commit. The vector in
|
||||||
|
# tests/vectors/key-identity is written only by the C# reference; this walks every row against the Python cache.
|
||||||
|
VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "key-identity"
|
||||||
|
|
||||||
|
IDS = {
|
||||||
|
"alpha": "sha256:b20703a98f49c253898404ab50ba2c630720f0b8a5e1a41614d5650be22799fe",
|
||||||
|
"beta": "sha256:dee4b727cf2ee04e61ab635ff0c5f3dca7345a411dc5f1018e583ab16009cf94",
|
||||||
|
"gamma": "sha256:fbc26e351e00c9e7185e8de171c7486aba707c229c949f0aae6b889288af5f6d",
|
||||||
|
}
|
||||||
|
DOCUMENTS = {
|
||||||
|
label: define_database_entry_type(
|
||||||
|
f"tests.ki_{label}",
|
||||||
|
[database_entry_field("text", 0, codec=t.string)],
|
||||||
|
schema_id=schema_id,
|
||||||
|
schema_name=f"tests.ki_{label}",
|
||||||
|
)
|
||||||
|
for label, schema_id in IDS.items()
|
||||||
|
}
|
||||||
|
LABEL_OF = {document.type: label for label, document in DOCUMENTS.items()}
|
||||||
|
|
||||||
|
def file_records(path: Path) -> list[list[str]]:
|
||||||
|
return sorted([envelope.key, envelope.type] for envelope in SingleFileMessagePackBackingStore(path).pull_all())
|
||||||
|
|
||||||
|
def stored(path: Path, key: str):
|
||||||
|
for envelope in SingleFileMessagePackBackingStore(path).pull_all():
|
||||||
|
if envelope.key == key:
|
||||||
|
return envelope.schema_id, envelope.stored_at, envelope.payload
|
||||||
|
return None
|
||||||
|
|
||||||
|
class KeyIdentityVectorTests(unittest.TestCase):
|
||||||
|
def test_rows(self) -> None:
|
||||||
|
rows = msgpack.unpackb((VECTORS / "rows.msgpack").read_bytes(), raw=False)
|
||||||
|
walked = 0
|
||||||
|
for label, registered, ops, expect in rows:
|
||||||
|
with self.subTest(row=label), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
self.walk(Path(tmp), registered, ops, expect)
|
||||||
|
walked += 1
|
||||||
|
self.assertEqual(walked, 12)
|
||||||
|
|
||||||
|
def walk(self, tmp: Path, registered: list[str], ops: list, expect: list) -> None:
|
||||||
|
refusal, held, file, foreign, kept = expect
|
||||||
|
state: dict = {"cache": None, "path": None, "before": None, "attached": []}
|
||||||
|
|
||||||
|
def builder() -> CultCache.builder:
|
||||||
|
made = CultCache.builder()
|
||||||
|
for name in registered:
|
||||||
|
made.register_document_type(DOCUMENTS[name])
|
||||||
|
return made
|
||||||
|
|
||||||
|
def open_cache(target) -> None:
|
||||||
|
if target == "memory":
|
||||||
|
state["cache"], state["path"] = builder().build(), None
|
||||||
|
return
|
||||||
|
path = tmp / "store.cc"
|
||||||
|
if target is not None and target != path:
|
||||||
|
shutil.copyfile(VECTORS / target, path)
|
||||||
|
state["before"] = path.read_bytes()
|
||||||
|
state["path"] = path
|
||||||
|
made = builder().add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||||
|
state["cache"] = made.build()
|
||||||
|
state["cache"].pull_all_backing_stores()
|
||||||
|
|
||||||
|
def perform(op: list) -> None:
|
||||||
|
cache = state["cache"]
|
||||||
|
if op[0] == "open":
|
||||||
|
open_cache(op[1])
|
||||||
|
elif op[0] == "attach":
|
||||||
|
path = tmp / f"attached-{len(state['attached'])}.cc"
|
||||||
|
shutil.copyfile(VECTORS / op[1], path)
|
||||||
|
state["attached"].append((path, path.read_bytes()))
|
||||||
|
cache.add_backing_store(
|
||||||
|
SingleFileMessagePackBackingStore(path), [DOCUMENTS[name].type for name in op[2]]
|
||||||
|
)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
elif op[0] == "reopen":
|
||||||
|
open_cache(state["path"]) # the same file, no fresh copy
|
||||||
|
elif op[0] == "put":
|
||||||
|
cache.put(DOCUMENTS[op[1]], op[2], {"text": op[3]})
|
||||||
|
elif op[0] == "delete":
|
||||||
|
cache.delete(DOCUMENTS[op[1]], op[2])
|
||||||
|
else:
|
||||||
|
self.fail(f"unknown op {op[0]}")
|
||||||
|
|
||||||
|
for op in ops[:-1]:
|
||||||
|
perform(op)
|
||||||
|
if refusal is None:
|
||||||
|
perform(ops[-1])
|
||||||
|
else:
|
||||||
|
key = refusal[0]
|
||||||
|
with self.assertRaises((SchemaConflictError, ValueError, RuntimeError)) as caught:
|
||||||
|
perform(ops[-1])
|
||||||
|
if ops[-1][0] == "attach":
|
||||||
|
# The key is read out of the refusal's own text (it opens "Record 'key' is held by") and occurs only as a
|
||||||
|
# delimited token, never inside a schema name or other word.
|
||||||
|
message = str(caught.exception)
|
||||||
|
named = re.match(r"Record '([^']*)' is held by", message)
|
||||||
|
self.assertIsNotNone(named, message)
|
||||||
|
self.assertEqual(named.group(1), key)
|
||||||
|
self.assertEqual(len(re.findall(rf"(?<![A-Za-z0-9_]){re.escape(key)}(?![A-Za-z0-9_])", message)), 1)
|
||||||
|
self.assertEqual(message.count(key), 1)
|
||||||
|
for path, before in state["attached"]:
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
else:
|
||||||
|
self.assertIsInstance(caught.exception, SchemaConflictError)
|
||||||
|
self.assertEqual(caught.exception.record_key, key)
|
||||||
|
self.assertIn(key, str(caught.exception))
|
||||||
|
cache = state["cache"]
|
||||||
|
got_held = sorted(
|
||||||
|
[LABEL_OF[type], key, value["text"]]
|
||||||
|
for type, values in cache.snapshot().items()
|
||||||
|
for key, value in values.items()
|
||||||
|
)
|
||||||
|
self.assertEqual(got_held, held)
|
||||||
|
path = state["path"]
|
||||||
|
if file is None:
|
||||||
|
self.assertIsNone(path)
|
||||||
|
else:
|
||||||
|
self.assertEqual(file_records(path), file)
|
||||||
|
self.assertEqual([record.key for record in cache.foreign_records()], foreign)
|
||||||
|
if kept:
|
||||||
|
vector = tmp / "vector-before.cc"
|
||||||
|
vector.write_bytes(state["before"])
|
||||||
|
for key in kept:
|
||||||
|
self.assertIsNotNone(stored(vector, key))
|
||||||
|
self.assertEqual(stored(path, key), stored(vector, key))
|
||||||
|
|
||||||
|
def alpha_beta(*names: str, store=None) -> CultCache:
|
||||||
|
made = CultCache.builder()
|
||||||
|
for name in names:
|
||||||
|
made.register_document_type(DOCUMENTS[name])
|
||||||
|
if store is not None:
|
||||||
|
made.add_generic_store(store)
|
||||||
|
return made.build()
|
||||||
|
|
||||||
|
class KeyIdentityTests(unittest.TestCase):
|
||||||
|
def test_two_global_types_are_refused_in_one_cache_and_a_store_holding_both_does_not_load(self) -> None:
|
||||||
|
# C# refuses a second persisted global on pull (cultcache-store-composition.md, "Globals never invented"); the Python
|
||||||
|
# JSONL store and a cache with no store used to let two global types share __global__, so such a file can exist.
|
||||||
|
ga = define_database_entry_type(
|
||||||
|
"tests.ki_ga", [database_entry_field("text", 0, codec=t.string)], schema_id="sha256:" + "1" * 64, global_document=True
|
||||||
|
)
|
||||||
|
gb = define_database_entry_type(
|
||||||
|
"tests.ki_gb", [database_entry_field("text", 0, codec=t.string)], schema_id="sha256:" + "2" * 64, global_document=True
|
||||||
|
)
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.jsonl"
|
||||||
|
for document in (ga, gb): # one cache per type that never pulled: how a file got two globals
|
||||||
|
writer = CultCache.builder().register_document_type(document).add_generic_store(JsonLinesBackingStore(path)).build()
|
||||||
|
writer.put_global(document, {"text": document.type})
|
||||||
|
both = CultCache.builder().register_document_type(ga).register_document_type(gb).add_generic_store(JsonLinesBackingStore(path)).build()
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
both.pull_all_backing_stores()
|
||||||
|
self.assertIn("'__global__' is held by both", str(caught.exception))
|
||||||
|
self.assertEqual(both.snapshot(), {})
|
||||||
|
memory = CultCache.builder().register_document_type(ga).register_document_type(gb).build()
|
||||||
|
memory.put_global(ga, {"text": "a"})
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
memory.put_global(gb, {"text": "b"})
|
||||||
|
self.assertEqual(memory.get_required_global(ga), {"text": "a"})
|
||||||
|
|
||||||
|
def test_every_write_path_refuses_a_second_type_and_never_echoes_the_payload(self) -> None:
|
||||||
|
alpha, beta = DOCUMENTS["alpha"], DOCUMENTS["beta"]
|
||||||
|
for name in ("put", "put_envelope", "put_envelopes"):
|
||||||
|
with self.subTest(path=name):
|
||||||
|
cache = alpha_beta("alpha", "beta")
|
||||||
|
cache.put(alpha, "k", {"text": "held"})
|
||||||
|
envelope = cache.get_envelope(alpha, "k")
|
||||||
|
other = type(envelope).create(
|
||||||
|
key="k", type=beta.type, payload=beta.encode_payload({"text": "CANARY"}), schema_id=IDS["beta"]
|
||||||
|
)
|
||||||
|
writes = {
|
||||||
|
"put": lambda: cache.put(beta, "k", {"text": "CANARY"}),
|
||||||
|
"put_envelope": lambda: cache.put_envelope(beta, other),
|
||||||
|
"put_envelopes": lambda: cache.put_envelopes(beta, [other]),
|
||||||
|
}
|
||||||
|
with self.assertRaises(SchemaConflictError) as caught:
|
||||||
|
writes[name]()
|
||||||
|
self.assertEqual(caught.exception.record_key, "k")
|
||||||
|
self.assertNotIn("CANARY", str(caught.exception))
|
||||||
|
self.assertEqual(cache.get(alpha, "k"), {"text": "held"})
|
||||||
|
self.assertIsNone(cache.get(beta, "k"))
|
||||||
|
|
||||||
|
def test_two_routed_stores_holding_one_key_refuse_the_pull_and_keep_the_earlier_state(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
one, two = Path(tmp) / "one.cc", Path(tmp) / "two.cc"
|
||||||
|
shutil.copyfile(VECTORS / "catalog-store.cc", one)
|
||||||
|
shutil.copyfile(VECTORS / "run-store.cc", two)
|
||||||
|
cache = alpha_beta("alpha", "beta")
|
||||||
|
cache.add_backing_store(SingleFileMessagePackBackingStore(one), [DOCUMENTS["alpha"].type])
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
cache.add_backing_store(SingleFileMessagePackBackingStore(two), [DOCUMENTS["beta"].type])
|
||||||
|
with self.assertRaises(Exception) as caught:
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertIn("'qx'", str(caught.exception))
|
||||||
|
self.assertEqual(cache.get(DOCUMENTS["alpha"], "qx"), {"text": "a"})
|
||||||
|
self.assertIsNone(cache.get(DOCUMENTS["beta"], "qx"))
|
||||||
|
|
||||||
|
def test_a_foreign_record_is_listed_and_its_bytes_survive_writes_and_removals_elsewhere(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.cc"
|
||||||
|
shutil.copyfile(VECTORS / "foreign-store.cc", path)
|
||||||
|
before = stored(path, "k")
|
||||||
|
cache = alpha_beta("alpha", store=SingleFileMessagePackBackingStore(path))
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
[record] = cache.foreign_records()
|
||||||
|
self.assertEqual((record.key, record.schema_id, record.schema_name), ("k", IDS["gamma"], "tests.ki_gamma"))
|
||||||
|
self.assertEqual(record.stored_at, before[1])
|
||||||
|
cache.put(DOCUMENTS["alpha"], "m", {"text": "m"})
|
||||||
|
cache.delete(DOCUMENTS["alpha"], "j")
|
||||||
|
self.assertEqual(stored(path, "k"), before)
|
||||||
|
self.assertEqual(file_records(path), [["k", "tests.ki_gamma"], ["m", "tests.ki_alpha"]])
|
||||||
|
claimed = alpha_beta("alpha", "gamma", store=SingleFileMessagePackBackingStore(path))
|
||||||
|
claimed.pull_all_backing_stores()
|
||||||
|
self.assertEqual(claimed.foreign_records(), [])
|
||||||
|
self.assertEqual(claimed.get(DOCUMENTS["gamma"], "k"), {"text": "g"})
|
||||||
|
|
||||||
|
def test_the_foreign_list_is_sorted_by_utf16_code_unit_and_replaced_by_the_next_pull(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.cc"
|
||||||
|
store = SingleFileMessagePackBackingStore(path)
|
||||||
|
gamma = DOCUMENTS["gamma"]
|
||||||
|
writer = alpha_beta("gamma", store=store)
|
||||||
|
for key in ("\U00010400", "A", "b", "B"):
|
||||||
|
writer.put(gamma, key, {"text": "x"})
|
||||||
|
cache = alpha_beta("alpha", store=SingleFileMessagePackBackingStore(path))
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual([record.key for record in cache.foreign_records()], ["B", "b", "\U00010400", "A"])
|
||||||
|
for key in ("B", "b", "\U00010400", "A"):
|
||||||
|
writer.delete(gamma, key)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(cache.foreign_records(), [])
|
||||||
|
|
||||||
|
class OwnershipAndStoreKindTests(unittest.TestCase):
|
||||||
|
"""The cache judge answers on its own where the store has no commit guard (JSONL) or the records live in two stores."""
|
||||||
|
|
||||||
|
def test_a_jsonl_store_gets_no_help_from_a_disk_judge(self) -> None:
|
||||||
|
from cultcache_py import JsonLinesBackingStore
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.jsonl"
|
||||||
|
gamma = DOCUMENTS["gamma"]
|
||||||
|
alpha_beta("gamma", store=JsonLinesBackingStore(path)).put(gamma, "k", {"text": "g"})
|
||||||
|
before = path.read_bytes()
|
||||||
|
cache = alpha_beta("alpha", store=JsonLinesBackingStore(path))
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual([record.key for record in cache.foreign_records()], ["k"])
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.put(DOCUMENTS["alpha"], "k", {"text": "x"})
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.delete(DOCUMENTS["alpha"], "k")
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
cache.put(DOCUMENTS["alpha"], "j", {"text": "a"}) # the foreign line is carried through a write elsewhere
|
||||||
|
self.assertIn(b"tests.ki_gamma", path.read_bytes())
|
||||||
|
|
||||||
|
def test_a_foreign_key_in_two_stores_refuses_the_pull(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
one, two = Path(tmp) / "one.cc", Path(tmp) / "two.cc"
|
||||||
|
for path in (one, two):
|
||||||
|
alpha_beta("gamma", store=SingleFileMessagePackBackingStore(path)).put(DOCUMENTS["gamma"], "k", {"text": "g"})
|
||||||
|
cache = alpha_beta("alpha")
|
||||||
|
cache.add_backing_store(SingleFileMessagePackBackingStore(one), [DOCUMENTS["alpha"].type])
|
||||||
|
cache.add_backing_store(SingleFileMessagePackBackingStore(two), [DOCUMENTS["beta"].type])
|
||||||
|
with self.assertRaises(Exception) as caught:
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertIn("'k'", str(caught.exception))
|
||||||
|
|
||||||
|
def test_a_type_that_owns_the_held_schema_id_replaces_the_record_and_the_key_keeps_one_type(self) -> None:
|
||||||
|
successor = define_database_entry_type(
|
||||||
|
"tests.ki_alpha2",
|
||||||
|
[database_entry_field("text", 0, codec=t.string)],
|
||||||
|
schema_id="sha256:" + "a" * 64,
|
||||||
|
schema_name="tests.ki_alpha2",
|
||||||
|
compatible_schema_ids=[IDS["alpha"]],
|
||||||
|
)
|
||||||
|
for routed in (False, True):
|
||||||
|
with self.subTest(routed=routed), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
one, two = Path(tmp) / "one.cc", Path(tmp) / "two.cc"
|
||||||
|
made = CultCache.builder().register_document_type(DOCUMENTS["alpha"]).register_document_type(successor)
|
||||||
|
if routed:
|
||||||
|
made.add_backing_store(SingleFileMessagePackBackingStore(one), [DOCUMENTS["alpha"].type])
|
||||||
|
made.add_backing_store(SingleFileMessagePackBackingStore(two), [successor.type])
|
||||||
|
else:
|
||||||
|
made.add_generic_store(SingleFileMessagePackBackingStore(one))
|
||||||
|
cache = made.build()
|
||||||
|
cache.put(DOCUMENTS["alpha"], "k", {"text": "old"})
|
||||||
|
if routed:
|
||||||
|
with self.assertRaises(SchemaConflictError): # one key would live in two stores
|
||||||
|
cache.put(successor, "k", {"text": "new"})
|
||||||
|
self.assertEqual(cache.get(DOCUMENTS["alpha"], "k"), {"text": "old"})
|
||||||
|
else:
|
||||||
|
cache.put(successor, "k", {"text": "new"})
|
||||||
|
self.assertIsNone(cache.get(DOCUMENTS["alpha"], "k"))
|
||||||
|
self.assertEqual(cache.get(successor, "k"), {"text": "new"})
|
||||||
|
self.assertEqual(file_records(one), [["k", "tests.ki_alpha2"]])
|
||||||
492
packages/cultcache-py/tests/test_slot_readonly.py
Normal file
492
packages/cultcache-py/tests/test_slot_readonly.py
Normal file
|
|
@ -0,0 +1,492 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import shutil
|
||||||
|
import threading
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
from cultcache_py.cache import CultCacheError
|
||||||
|
|
||||||
|
from cultcache_py import (
|
||||||
|
CultCache,
|
||||||
|
CultCachePayloadDepthError,
|
||||||
|
SlotCodec,
|
||||||
|
SchemaConflictError,
|
||||||
|
SingleFileMessagePackBackingStore,
|
||||||
|
UnionArm,
|
||||||
|
UnreadSlots,
|
||||||
|
database_entry_field,
|
||||||
|
define_database_entry_type,
|
||||||
|
define_record_codec,
|
||||||
|
leaves as t,
|
||||||
|
)
|
||||||
|
import test_union_parity as _parity
|
||||||
|
from test_union_parity import (
|
||||||
|
VECTORS,
|
||||||
|
_surface_value,
|
||||||
|
_vector,
|
||||||
|
surface_document,
|
||||||
|
unit_outcome,
|
||||||
|
)
|
||||||
|
|
||||||
|
SLOT = "extra-slot" # the canary: the value of every extra slot in the vectors; no refusal may carry it
|
||||||
|
|
||||||
|
# The vectors name the C# types; this runtime names a type by its codec's name, a document by its document type.
|
||||||
|
_NAMES = {"UnionSurface": surface_document.type, "SurfaceDeclaring": surface_document.type, "Denied2": "Denied"}
|
||||||
|
|
||||||
|
slot_unit_document = define_database_entry_type(
|
||||||
|
"tests.slot_unit",
|
||||||
|
[database_entry_field("name", 0, codec=t.string), database_entry_field("outcome", 1, codec=unit_outcome)],
|
||||||
|
name="name",
|
||||||
|
)
|
||||||
|
# The reader of extra-slot-consent.cc: the surface document's slots under a new schema id that lists the old one compatible.
|
||||||
|
declaring_document = define_database_entry_type(
|
||||||
|
"tests.union_surface",
|
||||||
|
[
|
||||||
|
database_entry_field("id", 0, codec=t.string),
|
||||||
|
database_entry_field("version", 1, codec=t.int32),
|
||||||
|
database_entry_field("root", 2, codec=_parity._node),
|
||||||
|
database_entry_field("outcomes", 3, codec=t.list(_parity.outcome)),
|
||||||
|
database_entry_field("part", 4, codec=t.nullable(_parity.item)),
|
||||||
|
database_entry_field("crafted", 5, codec=t.nullable(_parity.crafted)),
|
||||||
|
database_entry_field("missing", 6, codec=_parity._node),
|
||||||
|
],
|
||||||
|
name="id",
|
||||||
|
schema_id="tests.union_surface.v2",
|
||||||
|
schema_name="tests.union_surface",
|
||||||
|
compatible_schema_ids=("tests.union_surface.wide",),
|
||||||
|
)
|
||||||
|
|
||||||
|
# file -> (type, slots carried, slots declared, every mark at depth 0): the README table.
|
||||||
|
_PAYLOADS = {
|
||||||
|
"extra-slot-record.msgpack": ("UnionSurface", 8, 7, True),
|
||||||
|
"extra-slot-nested-record.msgpack": ("Composition", 3, 2, False),
|
||||||
|
"extra-slot-arm.msgpack": ("Box", 3, 2, False),
|
||||||
|
"extra-slot-list-element.msgpack": ("Text", 3, 2, False),
|
||||||
|
"depth-skipped-at-limit.msgpack": ("UnionSurface", 8, 7, True),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_store_without_variants(source: Path, target: Path) -> None:
|
||||||
|
"""This runtime holds no variant records (a five-slot record makes a store unreadable here), so the store vector is
|
||||||
|
walked with its variant row removed; every other byte is the C# writer's."""
|
||||||
|
[snapshot] = list(msgpack.Unpacker(open(source, "rb"), raw=False))
|
||||||
|
snapshot[2] = [record for record in snapshot[2] if len(record) <= 4]
|
||||||
|
target.write_bytes(msgpack.packb(snapshot, use_bin_type=True))
|
||||||
|
|
||||||
|
|
||||||
|
def _open(path: Path, *documents) -> CultCache:
|
||||||
|
builder = CultCache.builder()
|
||||||
|
for document in documents:
|
||||||
|
builder.register_document_type(document)
|
||||||
|
cache = builder.add_generic_store(SingleFileMessagePackBackingStore(path)).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
return cache
|
||||||
|
|
||||||
|
|
||||||
|
def _rows():
|
||||||
|
return msgpack.unpackb(_vector("slot-outcomes.msgpack"), raw=False)
|
||||||
|
|
||||||
|
|
||||||
|
class PayloadMarkTests(unittest.TestCase):
|
||||||
|
def test_every_read_only_payload_decodes_and_names_its_mark(self) -> None:
|
||||||
|
for file, (type_name, carried, declared, depth0) in _PAYLOADS.items():
|
||||||
|
with self.subTest(file=file):
|
||||||
|
value, mark = surface_document.decode_payload_marked(_vector(file))
|
||||||
|
self.assertEqual(value, _surface_value())
|
||||||
|
self.assertEqual(mark, UnreadSlots(_NAMES.get(type_name, type_name), carried, declared, 1, depth0))
|
||||||
|
self.assertEqual(surface_document.decode_payload(_vector(file)), _surface_value())
|
||||||
|
|
||||||
|
def test_the_outcomes_manifest_still_counts_nine_rows_and_four_read_only(self) -> None:
|
||||||
|
rows = msgpack.unpackb(_vector("outcomes.msgpack"), raw=False)
|
||||||
|
self.assertEqual(len(rows), 9)
|
||||||
|
self.assertEqual([row[0] for row in rows if row[1] == "read-only"], [f for f in _PAYLOADS if f != "depth-skipped-at-limit.msgpack"])
|
||||||
|
|
||||||
|
def test_a_clean_payload_and_a_short_one_carry_no_mark(self) -> None:
|
||||||
|
self.assertEqual(surface_document.decode_payload_marked(_vector("union-surface.msgpack")), (_surface_value(), None))
|
||||||
|
slots = msgpack.unpackb(_vector("union-surface.msgpack"), raw=False)
|
||||||
|
slots[2] = [1, ["only-id"]] # a Text arm with a missing trailing slot: defaults, no mark
|
||||||
|
_, mark = surface_document.decode_payload_marked(msgpack.packb(slots, use_bin_type=True))
|
||||||
|
self.assertIsNone(mark)
|
||||||
|
|
||||||
|
def test_a_retired_slot_below_the_highest_key_does_not_mark(self) -> None:
|
||||||
|
slots = msgpack.unpackb(_vector("union-surface.msgpack"), raw=False)
|
||||||
|
# Image's declared slots are 0, 1, 3 (2 retired): four carried, four declared.
|
||||||
|
image = slots[2][1][1][2]
|
||||||
|
self.assertEqual(image[0], 5)
|
||||||
|
image[1][2] = "retired-slot-value"
|
||||||
|
_, mark = surface_document.decode_payload_marked(msgpack.packb(slots, use_bin_type=True))
|
||||||
|
self.assertIsNone(mark)
|
||||||
|
|
||||||
|
def test_the_depth_check_runs_before_the_mark(self) -> None:
|
||||||
|
with self.assertRaises(CultCachePayloadDepthError):
|
||||||
|
surface_document.decode_payload_marked(_vector("depth-skipped-past-limit.msgpack"))
|
||||||
|
|
||||||
|
def test_the_unit_arm_with_extra_slots_is_read_past_and_marked(self) -> None:
|
||||||
|
payload = msgpack.packb(["unit-arm", [2, [SLOT]]], use_bin_type=True)
|
||||||
|
value, mark = slot_unit_document.decode_payload_marked(payload)
|
||||||
|
self.assertEqual(value, {"name": "unit-arm", "outcome": UnionArm("Denied")})
|
||||||
|
self.assertEqual(mark, UnreadSlots("Denied", 1, 0, 1, False))
|
||||||
|
self.assertEqual(slot_unit_document.decode_payload_marked(msgpack.packb(["ok", [2, []]]))[1], None)
|
||||||
|
|
||||||
|
def test_a_codec_decoded_outside_a_payload_decode_notes_nothing(self) -> None:
|
||||||
|
self.assertEqual(unit_outcome.decode([2, [SLOT]]), UnionArm("Denied"))
|
||||||
|
|
||||||
|
def test_the_scope_belongs_to_its_thread(self) -> None:
|
||||||
|
def decode_elsewhere(raw):
|
||||||
|
thread = threading.Thread(target=lambda: unit_outcome.decode([2, [SLOT]]))
|
||||||
|
thread.start()
|
||||||
|
thread.join()
|
||||||
|
return raw
|
||||||
|
|
||||||
|
outer = define_database_entry_type("tests.threaded", [database_entry_field("a", 0, codec=SlotCodec(lambda v: v, decode_elsewhere))])
|
||||||
|
self.assertIsNone(outer.decode_payload_marked(msgpack.packb(["x"]))[1])
|
||||||
|
|
||||||
|
def test_a_nested_payload_decode_restores_the_outer_scope(self) -> None:
|
||||||
|
def decode_clean_inside(raw):
|
||||||
|
surface_document.decode_payload_marked(_vector("union-surface.msgpack"))
|
||||||
|
return raw
|
||||||
|
|
||||||
|
inner = define_record_codec("Inner", [database_entry_field("x", 0, codec=t.string)])
|
||||||
|
outer = define_database_entry_type(
|
||||||
|
"tests.nested",
|
||||||
|
[
|
||||||
|
database_entry_field("a", 0, codec=SlotCodec(lambda v: v, decode_clean_inside)),
|
||||||
|
database_entry_field("b", 1, codec=inner),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
_, mark = outer.decode_payload_marked(msgpack.packb(["x", ["v", SLOT]]))
|
||||||
|
self.assertEqual(mark, UnreadSlots("Inner", 2, 1, 1, False))
|
||||||
|
|
||||||
|
|
||||||
|
class StoreRowTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.tmp = Path(tempfile.mkdtemp())
|
||||||
|
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||||
|
|
||||||
|
def _store(self, name: str) -> tuple[Path, CultCache]:
|
||||||
|
path = self.tmp / name
|
||||||
|
_copy_store_without_variants(VECTORS / name, path)
|
||||||
|
documents = (declaring_document,) if name == "extra-slot-consent.cc" else (surface_document, slot_unit_document)
|
||||||
|
return path, _open(path, *documents)
|
||||||
|
|
||||||
|
def _document(self, store: str, key: str):
|
||||||
|
if key == "unit-arm":
|
||||||
|
return slot_unit_document
|
||||||
|
return declaring_document if store == "extra-slot-consent.cc" else surface_document
|
||||||
|
|
||||||
|
def test_every_row_is_writable_exactly_where_the_vector_says_and_refuses_with_its_mark(self) -> None:
|
||||||
|
walked = 0
|
||||||
|
for store, key, writable, explained, type_name, carried, declared, occurrences, flatten in _rows():
|
||||||
|
if flatten is not None:
|
||||||
|
continue # a variant: this runtime holds none (deviation)
|
||||||
|
walked += 1
|
||||||
|
with self.subTest(store=store, key=key):
|
||||||
|
path, cache = self._store(store)
|
||||||
|
document = self._document(store, key)
|
||||||
|
before = path.read_bytes()
|
||||||
|
self.assertEqual(cache.is_writable(document, key), writable)
|
||||||
|
held = cache.get(document, key)
|
||||||
|
self.assertIsNotNone(held)
|
||||||
|
if writable:
|
||||||
|
continue
|
||||||
|
self.assertEqual(explained, False)
|
||||||
|
for attempt in (
|
||||||
|
lambda: cache.put(document, key, held),
|
||||||
|
lambda: cache.delete(document, key),
|
||||||
|
lambda: cache.put_envelope(document, cache.get_required_envelope(document, key)),
|
||||||
|
lambda: cache.put_envelopes(document, [cache.get_required_envelope(document, key)]),
|
||||||
|
lambda: cache.update(document, key, lambda value: value),
|
||||||
|
):
|
||||||
|
with self.assertRaises(SchemaConflictError) as caught:
|
||||||
|
attempt()
|
||||||
|
refusal = caught.exception
|
||||||
|
self.assertEqual(
|
||||||
|
refusal.unread_slots,
|
||||||
|
UnreadSlots(_NAMES.get(type_name, type_name), carried, declared, occurrences, refusal.unread_slots.depth0_only),
|
||||||
|
)
|
||||||
|
self.assertEqual(refusal.record_key, key)
|
||||||
|
message = str(refusal)
|
||||||
|
self.assertNotIn(SLOT, message)
|
||||||
|
self.assertIn(refusal.unread_slots.type_name, message)
|
||||||
|
self.assertIn(f"{carried} slots", message)
|
||||||
|
self.assertIn(f"declares {declared}", message)
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
self.assertTrue(cache.get(document, key) is held)
|
||||||
|
self.assertEqual(walked, 7) # eight rows, the variant not walked
|
||||||
|
|
||||||
|
def test_a_key_no_store_holds_and_another_key_are_writable_while_a_marked_one_is_held(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "marked"))
|
||||||
|
self.assertTrue(cache.is_writable(surface_document, "nothing-here"))
|
||||||
|
cache.put(surface_document, "other", cache.get_required(surface_document, "clean"))
|
||||||
|
self.assertEqual(cache.get(surface_document, "other"), _surface_value())
|
||||||
|
|
||||||
|
def test_consent_lands_an_explained_mark_and_refuses_the_others(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-consent.cc")
|
||||||
|
self.assertTrue(cache.is_writable(declaring_document, "consented"))
|
||||||
|
cache.put(declaring_document, "consented", cache.get_required(declaring_document, "consented"))
|
||||||
|
[stored] = [e for e in SingleFileMessagePackBackingStore(path).pull_all() if e.key == "consented"]
|
||||||
|
self.assertEqual(stored.schema_id, "tests.union_surface.v2")
|
||||||
|
self.assertEqual(len(msgpack.unpackb(stored.payload, raw=False)), 7) # the consented slot is gone
|
||||||
|
self.assertTrue(cache.is_writable(declaring_document, "consented"))
|
||||||
|
for key in ("over-consent", "nested"):
|
||||||
|
self.assertFalse(cache.is_writable(declaring_document, key), key)
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.put(declaring_document, key, cache.get_required(declaring_document, key))
|
||||||
|
|
||||||
|
def test_a_reload_marks_again_and_a_write_of_a_clean_envelope_over_a_marked_key_is_refused(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
envelope = cache.get_required_envelope(surface_document, "clean")
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.put_envelope(surface_document, type(envelope)(**{**envelope.__dict__, "key": "marked"}))
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "marked"))
|
||||||
|
|
||||||
|
def test_a_written_envelope_that_carries_extra_slots_is_held_read_only(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
envelope = cache.get_required_envelope(surface_document, "clean")
|
||||||
|
carrying = type(envelope)(**{**envelope.__dict__, "key": "arrived", "payload": _vector("extra-slot-arm.msgpack")})
|
||||||
|
cache.put_envelope(surface_document, carrying)
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "arrived"))
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.delete(surface_document, "arrived")
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "arrived"))
|
||||||
|
|
||||||
|
def test_a_batch_of_envelopes_that_carry_extra_slots_is_held_read_only(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
envelope = cache.get_required_envelope(surface_document, "clean")
|
||||||
|
batch = [
|
||||||
|
type(envelope)(**{**envelope.__dict__, "key": key, "payload": _vector(file)})
|
||||||
|
for key, file in (("b1", "extra-slot-arm.msgpack"), ("b2", "union-surface.msgpack"))
|
||||||
|
]
|
||||||
|
cache.put_envelopes(surface_document, batch)
|
||||||
|
self.assertEqual([cache.is_writable(surface_document, key) for key in ("b1", "b2")], [False, True])
|
||||||
|
|
||||||
|
def test_a_mark_under_an_id_the_reader_does_not_list_compatible_is_not_consented(self) -> None:
|
||||||
|
unlisted = define_database_entry_type(
|
||||||
|
"tests.union_surface",
|
||||||
|
[database_entry_field("id", 0, codec=t.string)] + [database_entry_field(n, i, codec=c) for n, i, c in (
|
||||||
|
("version", 1, t.int32), ("root", 2, _parity._node), ("outcomes", 3, t.list(_parity.outcome)),
|
||||||
|
("part", 4, t.nullable(_parity.item)), ("crafted", 5, t.nullable(_parity.crafted)), ("missing", 6, _parity._node))],
|
||||||
|
name="id",
|
||||||
|
schema_id="tests.union_surface.v2",
|
||||||
|
schema_name="tests.union_surface",
|
||||||
|
)
|
||||||
|
path = self.tmp / "unlisted.cc"
|
||||||
|
_copy_store_without_variants(VECTORS / "extra-slot-consent.cc", path)
|
||||||
|
cache = _open(path, unlisted)
|
||||||
|
self.assertFalse(cache.is_writable(unlisted, "consented"))
|
||||||
|
|
||||||
|
def test_a_written_envelope_is_judged_on_the_id_it_is_stored_under_so_a_reload_agrees(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-consent.cc")
|
||||||
|
arrived = next(e for e in SingleFileMessagePackBackingStore(path).pull_all() if e.key == "consented")
|
||||||
|
self.assertEqual(arrived.schema_id, "tests.union_surface.wide")
|
||||||
|
cache.put_envelope(declaring_document, dataclasses.replace(arrived, key="via-envelope"))
|
||||||
|
cache.put_envelopes(declaring_document, [dataclasses.replace(arrived, key="via-batch")])
|
||||||
|
in_memory = [cache.is_writable(declaring_document, k) for k in ("via-envelope", "via-batch")]
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual([cache.is_writable(declaring_document, k) for k in ("via-envelope", "via-batch")], in_memory)
|
||||||
|
self.assertEqual(in_memory, [False, False])
|
||||||
|
|
||||||
|
def test_is_writable_agrees_with_the_write_for_a_key_a_store_carries_foreign(self) -> None:
|
||||||
|
path = self.tmp / "foreign.cc"
|
||||||
|
_copy_store_without_variants(VECTORS / "extra-slot-store.cc", path)
|
||||||
|
cache = _open(path, surface_document) # tests.slot_unit is not registered: its record is carried foreign
|
||||||
|
self.assertEqual([r.key for r in cache.foreign_records()], ["unit-arm"])
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "unit-arm"))
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.put(surface_document, "unit-arm", _surface_value())
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache.delete(surface_document, "unit-arm")
|
||||||
|
|
||||||
|
def test_every_key_of_a_batch_is_checked_wherever_the_marked_one_stands(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
before = path.read_bytes()
|
||||||
|
envelope = cache.get_required_envelope(surface_document, "clean")
|
||||||
|
fresh = dataclasses.replace(envelope, key="fresh")
|
||||||
|
marked = dataclasses.replace(envelope, key="marked")
|
||||||
|
for batch in ([fresh, marked], [marked, fresh], [fresh, dataclasses.replace(envelope, key="f2"), marked]):
|
||||||
|
with self.assertRaises(SchemaConflictError) as caught:
|
||||||
|
cache.put_envelopes(surface_document, batch)
|
||||||
|
self.assertEqual(caught.exception.record_key, "marked")
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
self.assertIsNone(cache.get(surface_document, "fresh"))
|
||||||
|
|
||||||
|
def test_only_the_marks_that_hold_a_record_read_only_are_kept_by_key(self) -> None:
|
||||||
|
_, store = self._store("extra-slot-store.cc")
|
||||||
|
self.assertEqual({k: v[0] for k, v in store._state.unread.items()}, {"marked": surface_document.type, "unit-arm": "tests.slot_unit"})
|
||||||
|
_, consent = self._store("extra-slot-consent.cc")
|
||||||
|
self.assertEqual(sorted(consent._state.unread), ["nested", "over-consent"]) # the consented mark is no state
|
||||||
|
|
||||||
|
def test_an_arrival_with_no_catalog_entry_under_a_listed_id_is_not_consented(self) -> None:
|
||||||
|
wide = next(e for e in SingleFileMessagePackBackingStore(VECTORS / "extra-slot-consent.cc").pull_all() if e.key == "consented")
|
||||||
|
bare = dataclasses.replace(wide, catalog_entry=None)
|
||||||
|
|
||||||
|
class Held:
|
||||||
|
def pull_all(self):
|
||||||
|
return [bare]
|
||||||
|
|
||||||
|
def push(self, envelope): ...
|
||||||
|
|
||||||
|
def push_all(self, envelopes): ...
|
||||||
|
|
||||||
|
def delete(self, type, key): ...
|
||||||
|
|
||||||
|
cache = CultCache.builder().register_document_type(declaring_document).add_generic_store(Held()).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertFalse(cache.is_writable(declaring_document, "consented"))
|
||||||
|
|
||||||
|
def test_a_removal_is_refused_where_is_writable_says_no(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
before = path.read_bytes()
|
||||||
|
other = define_database_entry_type("tests.other", [database_entry_field("a", 0, codec=t.string)], name="a")
|
||||||
|
cache2 = _open(self.tmp / "extra-slot-store.cc", surface_document, slot_unit_document, other)
|
||||||
|
self.assertFalse(cache2.is_writable(other, "clean"))
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
cache2.delete(other, "clean")
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
self.assertIsNotNone(cache2.get(surface_document, "clean"))
|
||||||
|
self.assertTrue(cache2.is_writable(other, "nothing-here"))
|
||||||
|
|
||||||
|
def test_a_global_document_is_not_writable_at_a_key_other_than_the_global_one(self) -> None:
|
||||||
|
solo = define_database_entry_type("tests.solo", [database_entry_field("a", 0, codec=t.string)], global_document=True)
|
||||||
|
cache = CultCache.builder().register_document_type(solo).build()
|
||||||
|
self.assertFalse(cache.is_writable(solo, "elsewhere"))
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.put(solo, "elsewhere", {"a": "x"})
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.delete(solo, "elsewhere")
|
||||||
|
self.assertTrue(cache.is_writable(solo, CultCache.GLOBAL_KEY))
|
||||||
|
|
||||||
|
def test_a_reload_replaces_the_marks_wholesale(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "marked"))
|
||||||
|
# The store is repaired by its owner: the marked record is rewritten clean. The reload must forget the old mark.
|
||||||
|
[snapshot] = list(msgpack.Unpacker(open(path, "rb"), raw=False))
|
||||||
|
clean = next(r for r in snapshot[2] if r[0] == "clean")
|
||||||
|
for record in snapshot[2]:
|
||||||
|
if record[0] in ("marked", "unit-arm"):
|
||||||
|
record[3] = clean[3] if record[0] == "marked" else record[3]
|
||||||
|
snapshot[2] = [r for r in snapshot[2] if r[0] != "unit-arm"]
|
||||||
|
path.write_bytes(msgpack.packb(snapshot, use_bin_type=True))
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertTrue(cache.is_writable(surface_document, "marked"))
|
||||||
|
self.assertEqual(cache._state.unread, {})
|
||||||
|
|
||||||
|
def test_a_listed_id_whose_catalog_reports_no_extra_slot_does_not_consent(self) -> None:
|
||||||
|
wide = next(e for e in SingleFileMessagePackBackingStore(VECTORS / "extra-slot-consent.cc").pull_all() if e.key == "consented")
|
||||||
|
same = dataclasses.replace(wide, catalog_entry=declaring_document.catalog_entry()) # the stored entry has no slot past the reader's
|
||||||
|
|
||||||
|
class Held:
|
||||||
|
def pull_all(self):
|
||||||
|
return [same]
|
||||||
|
|
||||||
|
def push(self, envelope): ...
|
||||||
|
|
||||||
|
def push_all(self, envelopes): ...
|
||||||
|
|
||||||
|
def delete(self, type, key): ...
|
||||||
|
|
||||||
|
cache = CultCache.builder().register_document_type(declaring_document).add_generic_store(Held()).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertFalse(cache.is_writable(declaring_document, "consented"))
|
||||||
|
|
||||||
|
def test_a_record_stored_under_the_readers_own_id_is_never_consented(self) -> None:
|
||||||
|
own = dataclasses.replace(
|
||||||
|
next(e for e in SingleFileMessagePackBackingStore(VECTORS / "extra-slot-consent.cc").pull_all() if e.key == "consented"),
|
||||||
|
schema_id="tests.union_surface.v2",
|
||||||
|
)
|
||||||
|
wider = dataclasses.replace(own.catalog_entry, schema_id="tests.union_surface.v2", compatible_schema_ids=("tests.union_surface.v2",))
|
||||||
|
listing = dataclasses.replace(
|
||||||
|
declaring_document, schema_id="tests.union_surface.v2", compatible_schema_ids=("tests.union_surface.wide", "tests.union_surface.v2")
|
||||||
|
)
|
||||||
|
|
||||||
|
class Held:
|
||||||
|
def pull_all(self):
|
||||||
|
return [dataclasses.replace(own, catalog_entry=wider)]
|
||||||
|
|
||||||
|
def push(self, envelope): ...
|
||||||
|
|
||||||
|
def push_all(self, envelopes): ...
|
||||||
|
|
||||||
|
def delete(self, type, key): ...
|
||||||
|
|
||||||
|
cache = CultCache.builder().register_document_type(listing).add_generic_store(Held()).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertFalse(cache.is_writable(listing, "consented"))
|
||||||
|
|
||||||
|
def test_a_removal_by_the_owner_of_a_key_held_as_another_type_removes_it_as_a_write_would_replace_it(self) -> None:
|
||||||
|
a = define_database_entry_type(
|
||||||
|
"tests.owned_a", [database_entry_field("a", 0, codec=t.string)], name="a", indexes={"by_a": "a"}
|
||||||
|
)
|
||||||
|
b = define_database_entry_type(
|
||||||
|
"tests.owned_b",
|
||||||
|
[database_entry_field("a", 0, codec=t.string)],
|
||||||
|
compatible_schema_ids=(a.catalog_entry().schema_id,),
|
||||||
|
)
|
||||||
|
path = self.tmp / "owned.cc"
|
||||||
|
writer = _open(path, a)
|
||||||
|
writer.put(a, "k", {"a": "x"})
|
||||||
|
writer.put(a, "other", {"a": "y"})
|
||||||
|
cache = _open(path, a, b)
|
||||||
|
self.assertTrue(cache.is_writable(b, "k"))
|
||||||
|
cache.delete(b, "k")
|
||||||
|
self.assertIsNone(cache.get(a, "k"))
|
||||||
|
self.assertIsNone(cache.get_by_index(a, "by_a", "x"))
|
||||||
|
self.assertEqual([e.key for e in SingleFileMessagePackBackingStore(path).pull_all()], ["other"])
|
||||||
|
self.assertTrue(cache.is_writable(b, "k"))
|
||||||
|
|
||||||
|
def test_an_owners_removal_is_one_store_call_so_the_cache_and_the_store_agree_when_it_fails(self) -> None:
|
||||||
|
a = define_database_entry_type("tests.one_a", [database_entry_field("a", 0, codec=t.string)], name="a")
|
||||||
|
b = define_database_entry_type(
|
||||||
|
"tests.one_b", [database_entry_field("a", 0, codec=t.string)], compatible_schema_ids=(a.catalog_entry().schema_id,)
|
||||||
|
)
|
||||||
|
path = self.tmp / "one.cc"
|
||||||
|
_open(path, a).put(a, "k", {"a": "x"})
|
||||||
|
|
||||||
|
class SecondDeleteFails(SingleFileMessagePackBackingStore):
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def delete(self, type, key):
|
||||||
|
self.calls += 1
|
||||||
|
if self.calls > 1:
|
||||||
|
raise OSError("second delete")
|
||||||
|
super().delete(type, key)
|
||||||
|
|
||||||
|
store = SecondDeleteFails(path)
|
||||||
|
cache = CultCache.builder().register_document_type(a).register_document_type(b).add_generic_store(store).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
cache.delete(b, "k")
|
||||||
|
self.assertEqual(store.calls, 1)
|
||||||
|
self.assertIsNone(cache.get(a, "k"))
|
||||||
|
self.assertEqual(SingleFileMessagePackBackingStore(path).pull_all(), [])
|
||||||
|
|
||||||
|
def test_writing_another_record_and_flushing_carries_the_marked_one_unchanged(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
original = {e.key: e for e in SingleFileMessagePackBackingStore(path).pull_all()}
|
||||||
|
cache.put(surface_document, "clean", {**_surface_value(), "version": 9})
|
||||||
|
after = {e.key: e for e in SingleFileMessagePackBackingStore(path).pull_all()}
|
||||||
|
for key in ("marked", "unit-arm"):
|
||||||
|
self.assertEqual(after[key].payload, original[key].payload, key)
|
||||||
|
self.assertEqual(after[key].schema_id, original[key].schema_id, key)
|
||||||
|
self.assertEqual(after[key].stored_at, original[key].stored_at, key)
|
||||||
|
self.assertNotEqual(after["clean"].payload, original["clean"].payload)
|
||||||
|
self.assertEqual(after["marked"].catalog_entry, original["marked"].catalog_entry)
|
||||||
|
|
||||||
|
def test_the_vector_store_loads_under_the_reference_documents_and_marks_nothing_else(self) -> None:
|
||||||
|
path, cache = self._store("extra-slot-store.cc")
|
||||||
|
self.assertTrue(cache.is_writable(surface_document, "clean"))
|
||||||
|
self.assertFalse(cache.is_writable(surface_document, "marked"))
|
||||||
|
self.assertFalse(cache.is_writable(slot_unit_document, "unit-arm"))
|
||||||
|
self.assertEqual(
|
||||||
|
cache.get(slot_unit_document, "unit-arm"), {"name": "unit-arm", "outcome": UnionArm("Denied")}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -261,7 +261,9 @@ class UnionSurfaceVectorTests(unittest.TestCase):
|
||||||
else:
|
else:
|
||||||
read_only += 1
|
read_only += 1
|
||||||
self.assertEqual(outcome_kind, "read-only", file)
|
self.assertEqual(outcome_kind, "read-only", file)
|
||||||
self.assertEqual(surface_document.decode_payload(payload), _surface_value(), file)
|
value, mark = surface_document.decode_payload_marked(payload)
|
||||||
|
self.assertEqual(value, _surface_value(), file)
|
||||||
|
self.assertIsNotNone(mark, file) # read past and marked: tests/test_slot_readonly.py pins what the mark holds
|
||||||
self.assertEqual((refused, read_only), (5, 4))
|
self.assertEqual((refused, read_only), (5, 4))
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -526,7 +528,9 @@ class BoundsVectorTests(unittest.TestCase):
|
||||||
surface_document.encode_payload(surface_document.decode_payload(payload)), _vector(detail), file
|
surface_document.encode_payload(surface_document.decode_payload(payload)), _vector(detail), file
|
||||||
)
|
)
|
||||||
elif outcome_kind == "read-only":
|
elif outcome_kind == "read-only":
|
||||||
self.assertEqual(surface_document.decode_payload(payload), _surface_value(), file)
|
value, mark = surface_document.decode_payload_marked(payload)
|
||||||
|
self.assertEqual(value, _surface_value(), file)
|
||||||
|
self.assertIsNotNone(mark, file)
|
||||||
else:
|
else:
|
||||||
with self.assertRaises(ValueError, msg=file) as caught:
|
with self.assertRaises(ValueError, msg=file) as caught:
|
||||||
surface_document.decode_payload(payload)
|
surface_document.decode_payload(payload)
|
||||||
|
|
|
||||||
664
packages/cultcache-py/tests/test_unique_index.py
Normal file
664
packages/cultcache-py/tests/test_unique_index.py
Normal file
|
|
@ -0,0 +1,664 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from decimal import Decimal
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
|
||||||
|
from cultcache_py import (
|
||||||
|
CultCache,
|
||||||
|
CultCacheEnvelope,
|
||||||
|
SingleFileMessagePackBackingStore,
|
||||||
|
define_database_entry_type,
|
||||||
|
define_document_type,
|
||||||
|
leaves as t,
|
||||||
|
database_entry_field,
|
||||||
|
)
|
||||||
|
from cultcache_py.cache import CultCacheError, index_text
|
||||||
|
|
||||||
|
# variants:ruling:unique-indexes. `indexes` are unique within one type, `roles` are selection aliases. The vectors in
|
||||||
|
# tests/vectors/unique-index are written by the C# reference; the rest of the behaviour is pinned here.
|
||||||
|
VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "unique-index"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Item:
|
||||||
|
sku: object = None
|
||||||
|
serial: object = None
|
||||||
|
group: object = None
|
||||||
|
|
||||||
|
|
||||||
|
def refusal(key_a: str, key_b: str, schema_name: str, alias: str) -> str:
|
||||||
|
return f"Records {key_a} and {key_b} ({schema_name}) both hold unique index '{alias}'; an index value names one record."
|
||||||
|
|
||||||
|
|
||||||
|
def item_doc(type: str = "item", *, indexes=None, roles=None):
|
||||||
|
return define_document_type(
|
||||||
|
type,
|
||||||
|
encode=asdict,
|
||||||
|
decode=lambda raw: Item(**raw),
|
||||||
|
indexes={"sku": "sku", "serial": "serial"} if indexes is None else indexes,
|
||||||
|
roles={"group": "group"} if roles is None else roles,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def envelope(document, key: str, item: Item) -> CultCacheEnvelope:
|
||||||
|
return CultCacheEnvelope.create(key=key, type=document.type, payload=document.encode_payload(item))
|
||||||
|
|
||||||
|
|
||||||
|
def cache_of(*documents, store=None) -> CultCache:
|
||||||
|
builder = CultCache.builder()
|
||||||
|
for document in documents:
|
||||||
|
builder.register_document_type(document)
|
||||||
|
if store is not None:
|
||||||
|
builder.add_generic_store(store)
|
||||||
|
return builder.build()
|
||||||
|
|
||||||
|
|
||||||
|
class RecordingStore:
|
||||||
|
"""A store that counts what reaches it; the judge must refuse before any store call."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.pushed: list[str] = []
|
||||||
|
|
||||||
|
def pull_all(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def push(self, envelope) -> None:
|
||||||
|
self.pushed.append(envelope.key)
|
||||||
|
|
||||||
|
def push_all(self, envelopes) -> None:
|
||||||
|
self.pushed.extend(envelope.key for envelope in envelopes)
|
||||||
|
|
||||||
|
def delete(self, type, key) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class IndexTextTests(unittest.TestCase):
|
||||||
|
def test_canonical_text(self) -> None:
|
||||||
|
cases = [
|
||||||
|
("S-1", "S-1"),
|
||||||
|
(" a ", " a "),
|
||||||
|
("", ""),
|
||||||
|
("\x1f", "\x1f"),
|
||||||
|
("\x1c", "\x1c"),
|
||||||
|
("\x1d", "\x1d"),
|
||||||
|
("\x1e", "\x1e"),
|
||||||
|
("", ""),
|
||||||
|
("", ""),
|
||||||
|
(True, "True"),
|
||||||
|
(False, "False"),
|
||||||
|
(7, "7"),
|
||||||
|
(-7, "-7"),
|
||||||
|
(9223372036854775807, "9223372036854775807"),
|
||||||
|
(18446744073709551615, "18446744073709551615"),
|
||||||
|
(0.0, "0"),
|
||||||
|
(-0.0, "0"),
|
||||||
|
(2.0, "2"),
|
||||||
|
(-3.0, "-3"),
|
||||||
|
(1e21, "1000000000000000000000"),
|
||||||
|
(0.1, "0.1000000000000000055511151231257827021181583404541015625"),
|
||||||
|
(-0.5, "-0.5"),
|
||||||
|
(2.5e-07, "0.00000024999999999999998868702795647156467140348468092270195484161376953125"),
|
||||||
|
]
|
||||||
|
for value, text in cases:
|
||||||
|
with self.subTest(value=value):
|
||||||
|
self.assertEqual(index_text(value), text)
|
||||||
|
self.assertIsInstance(index_text(value), str)
|
||||||
|
|
||||||
|
def test_no_value(self) -> None:
|
||||||
|
blank = ["", " ", "\t", "\n", "\x0b", "\x0c", "\r", "\x85", "\xa0", " ", "
", "
", " ",
|
||||||
|
" ", " ", " \x85 ", *map(chr, range(0x2000, 0x200B))]
|
||||||
|
for value in [None, float("nan"), float("inf"), float("-inf"), *blank]:
|
||||||
|
with self.subTest(value=repr(value)):
|
||||||
|
self.assertIsNone(index_text(value))
|
||||||
|
|
||||||
|
def test_blank_set_is_exactly_the_csharp_set(self) -> None:
|
||||||
|
csharp = {*range(0x09, 0x0E), 0x20, 0x85, 0xA0, 0x1680, *range(0x2000, 0x200B), 0x2028, 0x2029, 0x202F, 0x205F, 0x3000}
|
||||||
|
self.assertEqual(len(csharp), 25)
|
||||||
|
got = {code for code in range(0x110000) if 0xD800 > code or code > 0xDFFF if index_text(chr(code)) is None}
|
||||||
|
self.assertEqual(got, csharp)
|
||||||
|
# Python's isspace is the wrong test: it also counts U+001C-001F.
|
||||||
|
self.assertTrue("\x1f".isspace())
|
||||||
|
self.assertEqual(index_text("\x1f"), "\x1f")
|
||||||
|
|
||||||
|
def test_floats_render_exactly_not_through_str(self) -> None:
|
||||||
|
for value in (0.1, 1e-7, 123456789.123456789, 5e-324, 1.7976931348623157e308):
|
||||||
|
with self.subTest(value=value):
|
||||||
|
text = index_text(value)
|
||||||
|
self.assertNotIn("e", text.lower())
|
||||||
|
self.assertEqual(Decimal(text), Decimal(value))
|
||||||
|
self.assertFalse(text.endswith("0") and "." in text)
|
||||||
|
|
||||||
|
def test_bool_is_not_an_integer(self) -> None:
|
||||||
|
self.assertEqual(index_text(True), "True")
|
||||||
|
self.assertNotEqual(index_text(True), "1")
|
||||||
|
|
||||||
|
|
||||||
|
class UniqueIndexTests(unittest.TestCase):
|
||||||
|
def test_a_write_of_a_held_value_at_another_key_is_refused_by_every_write_path(self) -> None:
|
||||||
|
for alias in ("sku", "serial"): # two unique indexes: each is judged
|
||||||
|
other = "serial" if alias == "sku" else "sku"
|
||||||
|
paths = {
|
||||||
|
"put": lambda cache, d, k, i: cache.put(d, k, i),
|
||||||
|
"put_envelope": lambda cache, d, k, i: cache.put_envelope(d, envelope(d, k, i)),
|
||||||
|
"put_envelopes": lambda cache, d, k, i: cache.put_envelopes(d, [envelope(d, k, i)]),
|
||||||
|
"update": lambda cache, d, k, i: cache.update(d, k, lambda _held: i),
|
||||||
|
}
|
||||||
|
for name, write in paths.items():
|
||||||
|
with self.subTest(alias=alias, path=name):
|
||||||
|
document = item_doc()
|
||||||
|
store = RecordingStore()
|
||||||
|
cache = cache_of(document, store=store)
|
||||||
|
cache.put(document, "item:b", Item(sku="CANARY-b", serial="canary-b"))
|
||||||
|
cache.put(document, "item:a", Item(sku="CANARY-a", serial="canary-a"))
|
||||||
|
before = cache.snapshot()
|
||||||
|
pushed = list(store.pushed)
|
||||||
|
taken = "CANARY-b" if alias == "sku" else "canary-b"
|
||||||
|
landing = Item(**{alias: taken, other: "fresh"})
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
write(cache, document, "item:a", landing)
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", alias))
|
||||||
|
self.assertNotIn("anary", str(caught.exception))
|
||||||
|
self.assertEqual(cache.snapshot(), before)
|
||||||
|
self.assertEqual(store.pushed, pushed)
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, alias, taken), "item:b")
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, other, "fresh"))
|
||||||
|
|
||||||
|
def test_a_record_rewriting_its_own_key_keeps_its_value(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:a", Item(sku="S", serial="1", group="g"))
|
||||||
|
cache.put(document, "item:a", Item(sku="S", serial="2", group="h"))
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:a")
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "serial", "1"))
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "serial", "2"), "item:a")
|
||||||
|
|
||||||
|
def test_the_refusal_names_the_schema_not_the_type(self) -> None:
|
||||||
|
document = define_document_type(
|
||||||
|
"t.item", indexes={"sku": "sku"}, schema_name="schema.Item", schema_id="schema.Item.v1"
|
||||||
|
)
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "k:1", {"sku": "S"})
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.put(document, "k:2", {"sku": "S"})
|
||||||
|
self.assertEqual(str(caught.exception), refusal("k:1", "k:2", "schema.Item", "sku"))
|
||||||
|
|
||||||
|
def test_the_two_smallest_keys_are_named_in_ordinal_order(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:m", Item(sku="S"))
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.put_envelopes(document, [envelope(document, "item:z", Item(sku="S")), envelope(document, "item:A", Item(sku="S"))])
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:A", "item:m", "item", "sku"))
|
||||||
|
|
||||||
|
def test_put_envelopes_holding_one_value_twice_is_refused_whole(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
store = RecordingStore()
|
||||||
|
cache = cache_of(document, store=store)
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.put_envelopes(
|
||||||
|
document,
|
||||||
|
[
|
||||||
|
envelope(document, "item:b", Item(sku="S")),
|
||||||
|
envelope(document, "item:c", Item(sku="T")),
|
||||||
|
envelope(document, "item:a", Item(sku="S")),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "sku"))
|
||||||
|
self.assertEqual(cache.snapshot(), {})
|
||||||
|
self.assertEqual(store.pushed, [])
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "sku", "T"))
|
||||||
|
|
||||||
|
def test_verdict_does_not_depend_on_batch_order(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
rows = [("item:a", Item(sku="S", serial="1")), ("item:b", Item(sku="T", serial="1")), ("item:c", Item(sku="U"))]
|
||||||
|
for order in itertools.permutations(rows):
|
||||||
|
with self.subTest(order=[key for key, _ in order]):
|
||||||
|
cache = cache_of(document)
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.put_envelopes(document, [envelope(document, key, item) for key, item in order])
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "serial"))
|
||||||
|
self.assertEqual(cache.snapshot(), {})
|
||||||
|
|
||||||
|
def test_put_envelopes_swapping_two_records_values_lands(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:a", Item(sku="S", serial="1"))
|
||||||
|
cache.put(document, "item:b", Item(sku="T", serial="2"))
|
||||||
|
cache.put_envelopes(
|
||||||
|
document,
|
||||||
|
[envelope(document, "item:a", Item(sku="T", serial="2")), envelope(document, "item:b", Item(sku="S", serial="1"))],
|
||||||
|
)
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "T"), "item:a")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "serial", "1"), "item:b")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "serial", "2"), "item:a")
|
||||||
|
|
||||||
|
def test_a_batch_may_take_a_value_the_batch_releases(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:a", Item(sku="S"))
|
||||||
|
cache.put_envelopes(document, [envelope(document, "item:a", Item(sku="T")), envelope(document, "item:b", Item(sku="S"))])
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "T"), "item:a")
|
||||||
|
|
||||||
|
def test_delete_frees_the_value(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:a", Item(sku="S"))
|
||||||
|
cache.delete(document, "item:a")
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "sku", "S"))
|
||||||
|
cache.put(document, "item:b", Item(sku="S"))
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b")
|
||||||
|
|
||||||
|
def test_blank_values_never_conflict(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
blanks = ["", " ", "\x85", " ", None]
|
||||||
|
for number, blank in enumerate(blanks):
|
||||||
|
cache.put(document, f"item:blank{number}", Item(sku=blank, serial=blank))
|
||||||
|
cache.put_envelopes(document, [envelope(document, "item:more", Item(sku=" ")), envelope(document, "item:more2", Item(sku=""))])
|
||||||
|
for blank in ("", " ", "\x85"):
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "sku", blank))
|
||||||
|
# U+001F and U+FEFF are values.
|
||||||
|
cache.put(document, "item:us", Item(sku="\x1f"))
|
||||||
|
cache.put(document, "item:bom", Item(sku=""))
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "\x1f"), "item:us")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", ""), "item:bom")
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.put(document, "item:us2", Item(sku="\x1f"))
|
||||||
|
|
||||||
|
def test_values_compare_as_the_canonical_text(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:a", Item(sku=1))
|
||||||
|
for same in (1, 1.0, "1"):
|
||||||
|
with self.subTest(same=same), self.assertRaises(CultCacheError):
|
||||||
|
cache.put(document, "item:b", Item(sku=same))
|
||||||
|
cache.put(document, "item:true", Item(sku=True))
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "True"), "item:true")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "1"), "item:a")
|
||||||
|
cache.put(document, "item:f", Item(sku=0.1))
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "0.1000000000000000055511151231257827021181583404541015625"), "item:f")
|
||||||
|
|
||||||
|
def test_two_types_may_share_an_alias_and_a_value(self) -> None:
|
||||||
|
one, two = item_doc("one"), item_doc("two")
|
||||||
|
cache = cache_of(one, two)
|
||||||
|
cache.put(one, "k:1", Item(sku="S", serial="1"))
|
||||||
|
cache.put(two, "k:2", Item(sku="S", serial="1"))
|
||||||
|
self.assertEqual(cache.get_key_by_index(one, "sku", "S"), "k:1")
|
||||||
|
self.assertEqual(cache.get_key_by_index(two, "sku", "S"), "k:2")
|
||||||
|
cache.delete(one, "k:1")
|
||||||
|
self.assertEqual(cache.get_key_by_index(two, "sku", "S"), "k:2")
|
||||||
|
|
||||||
|
def test_roles_may_share_values_and_are_not_looked_up(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:a", Item(sku="1", group="g"))
|
||||||
|
cache.put(document, "item:b", Item(sku="2", group="g"))
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "group", "g"))
|
||||||
|
self.assertIsNone(cache.get_by_index(document, "group", "g"))
|
||||||
|
|
||||||
|
def test_an_alias_in_both_indexes_and_roles_is_refused(self) -> None:
|
||||||
|
with self.assertRaisesRegex(ValueError, "both an index and a role"):
|
||||||
|
define_document_type("bad", indexes={"a": "a"}, roles={"a": "a"})
|
||||||
|
with self.assertRaisesRegex(ValueError, "both an index and a role"):
|
||||||
|
define_database_entry_type("bad", [("a", 0)], indexes={"a": "a"}, roles={"a": "a"})
|
||||||
|
define_document_type("fine", indexes={"a": "a"}, roles={"b": "a"})
|
||||||
|
|
||||||
|
def test_register_index_over_held_duplicates_installs_nothing(self) -> None:
|
||||||
|
document = item_doc(indexes={}, roles={})
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:b", Item(sku="S", serial="1"))
|
||||||
|
cache.put(document, "item:a", Item(sku="S", serial="2"))
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.register_index(document, "sku", "sku")
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:a", "item:b", "item", "sku"))
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "sku", "S"))
|
||||||
|
cache.register_index(document, "serial", "serial") # an installed alias is not poisoned by the refused one
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "serial", "2"), "item:a")
|
||||||
|
# Nothing was installed, so a write sharing the refused alias's value is still admitted.
|
||||||
|
cache.put(document, "item:c", Item(sku="S", serial="3"))
|
||||||
|
|
||||||
|
def test_registration_names_an_installed_alias_a_caller_mutated_into_a_collision(self) -> None:
|
||||||
|
document = item_doc(indexes={"sku": "sku"}, roles={})
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:x", Item(sku="1", serial="s1"))
|
||||||
|
cache.put(document, "item:y", Item(sku="2", serial="s2"))
|
||||||
|
cache.get_required(document, "item:y").sku = "1" # the cache hands out its held value
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.register_index(document, "serial", "serial")
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:x", "item:y", "item", "sku"))
|
||||||
|
|
||||||
|
def test_derive_names_the_ordinal_first_alias_then_value(self) -> None:
|
||||||
|
document = item_doc(indexes={}, roles={})
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:1", Item(sku="a", serial="p", group="x"))
|
||||||
|
cache.put(document, "item:2", Item(sku="a", serial="p", group="x"))
|
||||||
|
cache.put(document, "item:3", Item(sku="b", serial="q", group="y"))
|
||||||
|
cache.put(document, "item:4", Item(sku="b", serial="q", group="y"))
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.register_index(document, "serial", "serial")
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:1", "item:2", "item", "serial"))
|
||||||
|
|
||||||
|
def test_a_document_in_two_unique_indexes_loads_and_registers_through_every_path(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "items.cc"
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document, store=SingleFileMessagePackBackingStore(path))
|
||||||
|
cache.put(document, "item:a", Item(sku="S", serial="1"))
|
||||||
|
cache.put(document, "item:b", Item(sku="T", serial="2"))
|
||||||
|
# a store written by a looser definition, then opened by this one
|
||||||
|
loose = item_doc(indexes={}, roles={})
|
||||||
|
writer = cache_of(loose, store=SingleFileMessagePackBackingStore(path))
|
||||||
|
writer.pull_all_backing_stores()
|
||||||
|
writer.put(loose, "item:c", Item(sku="U", serial="2"))
|
||||||
|
before = path.read_bytes()
|
||||||
|
reader = cache_of(document, store=SingleFileMessagePackBackingStore(path))
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
reader.pull_all_backing_stores()
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:b", "item:c", "item", "serial"))
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
self.assertEqual(reader.snapshot(), {})
|
||||||
|
|
||||||
|
def test_a_refused_load_keeps_the_held_state(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "items.cc"
|
||||||
|
loose = item_doc(indexes={}, roles={})
|
||||||
|
writer = cache_of(loose, store=SingleFileMessagePackBackingStore(path))
|
||||||
|
writer.put(loose, "item:a", Item(sku="S"))
|
||||||
|
writer.put(loose, "item:b", Item(sku="S"))
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document, store=SingleFileMessagePackBackingStore(path))
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "sku", "S"))
|
||||||
|
self.assertEqual(cache.get_all(document), [])
|
||||||
|
|
||||||
|
|
||||||
|
class SimulationFactTests(unittest.TestCase):
|
||||||
|
def test_simulation_fact_catalog_entry_is_unchanged_by_indexes_to_roles(self) -> None:
|
||||||
|
import re
|
||||||
|
|
||||||
|
from cultnet_py.cultmesh_contracts import SIMULATION_FACT_DOCUMENT_TYPE, simulation_fact_document as document
|
||||||
|
|
||||||
|
aliases = {"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"}
|
||||||
|
self.assertEqual(document.roles, aliases)
|
||||||
|
self.assertEqual(document.indexes, {})
|
||||||
|
entry = document.catalog_entry()
|
||||||
|
snake = lambda name: re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower() # noqa: E731
|
||||||
|
twin = define_database_entry_type(
|
||||||
|
SIMULATION_FACT_DOCUMENT_TYPE,
|
||||||
|
[(snake(m.member_name), m.slot) for m in entry.members],
|
||||||
|
name="fact_id",
|
||||||
|
indexes=aliases,
|
||||||
|
schema_name=entry.schema_name,
|
||||||
|
schema_version=entry.schema_version,
|
||||||
|
)
|
||||||
|
self.assertEqual(entry.canonical_schema_json, twin.catalog_entry().canonical_schema_json)
|
||||||
|
self.assertEqual(entry.content_hash, twin.catalog_entry().content_hash)
|
||||||
|
self.assertEqual(entry.members, twin.catalog_entry().members)
|
||||||
|
self.assertEqual({m.index_alias for m in entry.members if m.index_alias}, set(aliases))
|
||||||
|
|
||||||
|
def test_facts_sharing_every_role_value_are_all_admitted(self) -> None:
|
||||||
|
from cultnet_py.cultmesh_contracts import CultMeshSimulationFact, simulation_fact_document as document
|
||||||
|
|
||||||
|
cache = cache_of(document)
|
||||||
|
for number in range(3):
|
||||||
|
fact = CultMeshSimulationFact.from_candidate(
|
||||||
|
{"shardId": "s", "shardEpoch": 1, "frame": 4, "subjectId": "x", "claimKind": "k", "claimHash": f"h{number}",
|
||||||
|
"witnessCount": 1, "supportWeight": 1.0, "totalWeight": 1.0, "confidence": 1.0}
|
||||||
|
)
|
||||||
|
cache.put(document, f"simulation:{number}", fact)
|
||||||
|
self.assertEqual(len(cache.get_all(document)), 3)
|
||||||
|
self.assertIsNone(cache.get_key_by_index(document, "shard", "s"))
|
||||||
|
|
||||||
|
|
||||||
|
class VectorTests(unittest.TestCase):
|
||||||
|
def test_values_rows(self) -> None:
|
||||||
|
document = item_doc(indexes={"sku": "sku"}, roles={})
|
||||||
|
for label, kind, value, text in msgpack.unpackb((VECTORS / "values.msgpack").read_bytes(), raw=False):
|
||||||
|
with self.subTest(label=label):
|
||||||
|
self.assertEqual(index_text(value), text)
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:1", Item(sku=value))
|
||||||
|
if text is None:
|
||||||
|
cache.put(document, "item:2", Item(sku=value)) # blank: two records commit
|
||||||
|
self.assertEqual(len(cache.get_all(document)), 2)
|
||||||
|
else:
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", text), "item:1")
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.put(document, "item:2", Item(sku=value))
|
||||||
|
|
||||||
|
def test_store_verdicts(self) -> None:
|
||||||
|
unit = define_database_entry_type
|
||||||
|
outcomes = msgpack.unpackb((VECTORS / "outcomes.msgpack").read_bytes(), raw=False)
|
||||||
|
self.assertEqual(len(outcomes), 3)
|
||||||
|
for name, alias, verdict, key_a, key_b in outcomes:
|
||||||
|
for roles_first in (False, True): # the verdict does not depend on declaration order
|
||||||
|
with self.subTest(file=name, alias=alias, roles_first=roles_first), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / name
|
||||||
|
shutil.copyfile(VECTORS / name, path)
|
||||||
|
before = path.read_bytes()
|
||||||
|
fields = [database_entry_field("sku", 0, codec=t.string), database_entry_field("category", 1, codec=t.string)]
|
||||||
|
declared = {"indexes": {alias: "sku"} if alias else {}, "roles": {} if alias else {"Sku": "sku"}}
|
||||||
|
if roles_first:
|
||||||
|
declared = dict(reversed(list(declared.items())))
|
||||||
|
document = unit(
|
||||||
|
"tests.unique_item",
|
||||||
|
fields,
|
||||||
|
schema_id="sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882",
|
||||||
|
schema_name="tests.unique_item",
|
||||||
|
**declared,
|
||||||
|
)
|
||||||
|
cache = cache_of(document, store=SingleFileMessagePackBackingStore(path))
|
||||||
|
if verdict == "open":
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertGreaterEqual(len(cache.get_all(document)), 2)
|
||||||
|
else:
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(str(caught.exception), refusal(key_a, key_b, "tests.unique_item", alias))
|
||||||
|
self.assertNotIn("S-1", str(caught.exception))
|
||||||
|
self.assertEqual(cache.snapshot(), {})
|
||||||
|
self.assertEqual(path.read_bytes(), before)
|
||||||
|
|
||||||
|
|
||||||
|
def test_refusals(self) -> None:
|
||||||
|
ids = {
|
||||||
|
"tests.unique_item": "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882",
|
||||||
|
"tests.uq_pair": "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60",
|
||||||
|
"tests.uq_alias": "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f",
|
||||||
|
"tests.uq_wide": "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f",
|
||||||
|
"tests.uq_alpha": "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389",
|
||||||
|
"tests.uq_beta": "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586",
|
||||||
|
}
|
||||||
|
aliases = {
|
||||||
|
"tests.uq_pair": {"Sku": "sku", "Alt": "second"},
|
||||||
|
"tests.uq_alias": {"Sku": "sku", "alt": "second"},
|
||||||
|
"tests.uq_wide": {"\uff21": "sku", "\U00010400": "second"}, # U+FF21 and U+10400: code-unit order puts the astral alias first
|
||||||
|
}
|
||||||
|
documents = {
|
||||||
|
name: define_database_entry_type(
|
||||||
|
name,
|
||||||
|
[database_entry_field("sku", 0, codec=t.string), database_entry_field("second", 1, codec=t.string)],
|
||||||
|
indexes=aliases.get(name, {"Sku": "sku"}),
|
||||||
|
roles={},
|
||||||
|
schema_id=schema_id,
|
||||||
|
schema_name=name,
|
||||||
|
)
|
||||||
|
for name, schema_id in ids.items()
|
||||||
|
}
|
||||||
|
rows = msgpack.unpackb((VECTORS / "refusals.msgpack").read_bytes(), raw=False)
|
||||||
|
walked = 0
|
||||||
|
for label, ops, expected in rows:
|
||||||
|
if any(op[0] == "commit" and len({schema for _, schema, _, _ in op[1]}) > 1 for op in ops):
|
||||||
|
continue # put_envelopes takes one type: no batch across types (rows 3 and 26)
|
||||||
|
with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "store.cc"
|
||||||
|
state: dict[str, CultCache] = {}
|
||||||
|
|
||||||
|
def perform(op) -> None:
|
||||||
|
if op[0] == "open":
|
||||||
|
if op[1] is not None:
|
||||||
|
shutil.copyfile(VECTORS / op[1], path)
|
||||||
|
state["cache"] = cache_of(*documents.values(), store=SingleFileMessagePackBackingStore(path))
|
||||||
|
state["cache"].pull_all_backing_stores()
|
||||||
|
elif op[0] == "pull":
|
||||||
|
shutil.copyfile(VECTORS / op[1], path)
|
||||||
|
state["cache"].pull_all_backing_stores()
|
||||||
|
else:
|
||||||
|
document = documents[op[1][0][1]]
|
||||||
|
state["cache"].put_envelopes(
|
||||||
|
document, [envelope(document, key, {"sku": sku, "second": second}) for key, _, sku, second in op[1]]
|
||||||
|
)
|
||||||
|
|
||||||
|
for op in ops[:-1]:
|
||||||
|
perform(op)
|
||||||
|
if expected is None:
|
||||||
|
perform(ops[-1])
|
||||||
|
else:
|
||||||
|
last = ops[-1]
|
||||||
|
if last[0] == "open":
|
||||||
|
before = (VECTORS / last[1]).read_bytes()
|
||||||
|
elif last[0] == "pull":
|
||||||
|
before = (VECTORS / last[1]).read_bytes()
|
||||||
|
else:
|
||||||
|
before = path.read_bytes() if path.exists() else b""
|
||||||
|
k1, k2, schema, alias = expected
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
perform(last)
|
||||||
|
self.assertEqual(str(caught.exception), refusal(k1, k2, schema, alias))
|
||||||
|
self.assertEqual(path.read_bytes() if path.exists() else b"", before)
|
||||||
|
walked += 1
|
||||||
|
self.assertEqual(walked, 35)
|
||||||
|
|
||||||
|
|
||||||
|
class ListStore(RecordingStore):
|
||||||
|
"""A store whose contents the test swaps between loads: another process rewrote the file."""
|
||||||
|
|
||||||
|
def __init__(self, envelopes=()) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.envelopes = list(envelopes)
|
||||||
|
|
||||||
|
def pull_all(self):
|
||||||
|
return list(self.envelopes)
|
||||||
|
|
||||||
|
|
||||||
|
class BatchAndOrderTests(unittest.TestCase):
|
||||||
|
def test_a_batch_naming_a_key_twice_lands_the_last_record_and_keeps_every_held_lookup(self) -> None:
|
||||||
|
# C# CultCacheBatch stages one operation per key, the last one named.
|
||||||
|
document = item_doc()
|
||||||
|
store = RecordingStore()
|
||||||
|
cache = cache_of(document, store=store)
|
||||||
|
cache.put(document, "item:h", Item(sku="S"))
|
||||||
|
store.pushed.clear()
|
||||||
|
cache.put_envelopes(
|
||||||
|
document, [envelope(document, "item:k", Item(sku="S")), envelope(document, "item:k", Item(sku="T"))]
|
||||||
|
)
|
||||||
|
self.assertEqual(store.pushed, ["item:k"])
|
||||||
|
self.assertEqual(cache.get_required(document, "item:k").sku, "T")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:h")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "T"), "item:k")
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.put(document, "item:z", Item(sku="S"))
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:h", "item:z", "item", "sku"))
|
||||||
|
|
||||||
|
def test_a_batch_naming_a_key_twice_is_judged_on_the_record_that_lands(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
cache.put(document, "item:h", Item(sku="S"))
|
||||||
|
# the first record for k would collide with h; only the last lands, so the batch is admitted
|
||||||
|
cache.put_envelopes(
|
||||||
|
document, [envelope(document, "item:k", Item(sku="S")), envelope(document, "item:k", Item(sku="T"))]
|
||||||
|
)
|
||||||
|
# and the reverse order lands the colliding record, which is refused
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.put_envelopes(
|
||||||
|
document, [envelope(document, "item:j", Item(sku="U")), envelope(document, "item:j", Item(sku="S"))]
|
||||||
|
)
|
||||||
|
self.assertNotIn("item:j", cache.snapshot().get("item", {}))
|
||||||
|
|
||||||
|
def test_a_load_judges_every_type_at_once_aliases_in_ordinal_order(self) -> None:
|
||||||
|
first = item_doc("first", indexes={"zeta": "sku"}, roles={})
|
||||||
|
second = item_doc("second", indexes={"alpha": "sku"}, roles={})
|
||||||
|
loose_first, loose_second = item_doc("first", indexes={}, roles={}), item_doc("second", indexes={}, roles={})
|
||||||
|
store = ListStore(
|
||||||
|
[envelope(loose_first, "f:1", Item(sku="S")), envelope(loose_first, "f:2", Item(sku="S"))]
|
||||||
|
+ [envelope(loose_second, "s:1", Item(sku="S")), envelope(loose_second, "s:2", Item(sku="S"))]
|
||||||
|
)
|
||||||
|
cache = cache_of(first, second, store=store)
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(str(caught.exception), refusal("s:1", "s:2", "second", "alpha"))
|
||||||
|
|
||||||
|
def test_aliases_sort_by_utf16_code_unit_not_code_point(self) -> None:
|
||||||
|
astral, bmp = "k\U0001F600", "k\uff01" # code points: bmp < astral; UTF-16 units: D83D < FF01
|
||||||
|
document = item_doc(indexes={bmp: "sku", astral: "serial"}, roles={})
|
||||||
|
cache = cache_of(document)
|
||||||
|
rows = [Item(sku="S", serial="T"), Item(sku="S", serial="T")]
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.put_envelopes(document, [envelope(document, f"item:{n}", row) for n, row in enumerate(rows)])
|
||||||
|
self.assertEqual(str(caught.exception), refusal("item:0", "item:1", "item", astral))
|
||||||
|
|
||||||
|
def test_a_load_names_the_least_violation_whatever_order_the_store_yields(self) -> None:
|
||||||
|
alpha, beta = item_doc("alpha", indexes={"k": "sku"}, roles={}), item_doc("beta", indexes={"k": "sku"}, roles={})
|
||||||
|
loose_alpha, loose_beta = item_doc("alpha", indexes={}, roles={}), item_doc("beta", indexes={}, roles={})
|
||||||
|
store = ListStore(
|
||||||
|
[
|
||||||
|
envelope(loose_alpha, "a:0", Item(sku="Q")),
|
||||||
|
envelope(loose_beta, "b:1", Item(sku="S")),
|
||||||
|
envelope(loose_beta, "b:2", Item(sku="S")),
|
||||||
|
envelope(loose_alpha, "a:1", Item(sku="P")),
|
||||||
|
envelope(loose_alpha, "a:2", Item(sku="P")),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
cache = cache_of(alpha, beta, store=store)
|
||||||
|
with self.assertRaises(CultCacheError) as caught:
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(str(caught.exception), refusal("a:1", "a:2", "alpha", "k"))
|
||||||
|
|
||||||
|
def test_register_index_refuses_an_alias_the_definition_declares_as_a_role(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
cache = cache_of(document)
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.register_index(document, "group", "group")
|
||||||
|
cache.put_envelopes(
|
||||||
|
document, [envelope(document, "item:1", Item(group="guide")), envelope(document, "item:2", Item(group="guide"))]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class LoadFreshnessTests(unittest.TestCase):
|
||||||
|
def test_a_load_is_judged_on_the_file_alone_not_against_the_held_map(self) -> None:
|
||||||
|
document = item_doc()
|
||||||
|
store = ListStore([envelope(document, "item:b", Item(sku="S")), envelope(document, "item:c", Item(sku="T"))])
|
||||||
|
cache = cache_of(document, store=store)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:b")
|
||||||
|
# another process moved S from b to a, and b to T's old place
|
||||||
|
store.envelopes = [envelope(document, "item:a", Item(sku="S")), envelope(document, "item:b", Item(sku="U")), store.envelopes[1]]
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "S"), "item:a")
|
||||||
|
self.assertEqual(cache.get_key_by_index(document, "sku", "U"), "item:b")
|
||||||
|
|
||||||
|
|
||||||
|
class DefinitionTests(unittest.TestCase):
|
||||||
|
def test_every_constructor_refuses_an_alias_in_both_indexes_and_roles(self) -> None:
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
|
from cultcache_py import DocumentDefinition
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "both an index and a role"):
|
||||||
|
DocumentDefinition(type="item", indexes={"sku": "sku"}, roles={"sku": "sku"})
|
||||||
|
with self.assertRaisesRegex(ValueError, "both an index and a role"):
|
||||||
|
replace(item_doc(roles={}), roles={"sku": "sku"})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -8,12 +8,22 @@ only in git history.
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name.
|
- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
||||||
- A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3).
|
- A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3).
|
||||||
|
- `#[cultcache(index)]` means unique, as in C# `RefuseIndexSharing`: within one type and one alias no two records hold one non-blank value. A put, batch, push or load that would make two records hold a value is refused, naming both keys and the alias, and a store that already holds a duplicate does not open. A grouping index is no longer expressible; a consumer that relied on shared values must drop the `index` and look up by another way.
|
||||||
|
- `bool` index and lookup text renders `True` / `False`, as C# does, where it rendered `true` / `false`.
|
||||||
|
- `DatabaseEntry::UNIQUE_INDEXES` replaces `cult_indexes` as the one index declaration and changes type (its members are the unique aliases); a hand implementation overriding `cult_indexes` no longer compiles.
|
||||||
|
- `register_entry_type` over a type that already holds records re-judges them under the new declaration (unique indexes and the global rule) and refuses, changing nothing, when they violate it. A declaration with an index also refuses when a held record cannot be decoded as the new type; a declaration without one does not read the records, so it does not.
|
||||||
|
- A single-file store holds one record per key. The v1 and legacy readers refuse a store with two records under one key (`CultCacheStoreUnreadable`, the record `Undecodable`): the store does not open, and a write over it refuses and leaves the file. A consumer that holds such a file must repair it with the runtime that wrote it before opening it here.
|
||||||
|
- Every single-file writer (`push`, `push_all`, `insert_entry_if_absent`, the compare-exchange and snapshot-append family, and the cache's puts and batches) refuses to put a second type under a key the store holds, with `SchemaConflictError` naming the key and both types, writing nothing. Retyping a key is a delete, then an add (one batch may hold both). A write owns the held record when its schema id equals the id the held record is stored under, as C# `WrittenOwns` asks; a type renamed under a stable id therefore replaces its own record, and a write of another type name under another id is refused (`push` of the same type name under another id still replaces the record, as before; `compare_and_swap_entry`, and a `push_all` batch holding one type under two ids, refuse it). Registered types here declare no compatible ids, so equality is the whole test. Writers used to leave two records under the key.
|
||||||
|
- `compare_and_swap_entry` over a single file refuses an in-place retype (expected `A/k`, replacement `B/k` under another schema id) with the same `SchemaConflictError`, as `compare_exchange` does, where it used to retype the record and return `true`. A lost race still returns `false` first.
|
||||||
|
- A cache holds one record per key. `put`, `put_envelope`, `put_raw_envelope` and a batch of a second type under a key the cache holds, or a key a store carries foreign, are refused with `SchemaConflictError` even with no store (a memory-only cache used to hold both), whatever schema id the envelope claims: another registered type's record is never owned. `put_envelope`, `put_raw_envelope`, `put_prepared_batch` and `load_soa` also refuse, at any key, an envelope whose schema id names another type registered in this cache (the id a pull would resolve the record by). An id this cache does not register is trusted as the envelope claims it: at a key this cache does not hold, the single-file store decides ownership by that id, so such an envelope can replace another writer's record stored under it, and a reader registering the type the id names serves the record as that type. Store-level writers called directly (`push`, `insert_entry_if_absent`, compare-and-swap) still trust the claimed id. A refused write or pull changes nothing. Two stores holding one key refuse the pull, naming the key and both types, and so does one store holding two records at a key that resolve to one type. A store holding a record no registered type claims now loads (it used to refuse with "No schema is registered for persisted entry type") and the record is carried, through `put_prepared_batch` too, which lays foreign records back byte for byte; `delete` at a foreign key is refused. `load_soa` refuses a second type at a key or a record at a foreign key, and still refuses an unregistered type. The redb stores still key by `(type, key)`, and their own writers (`push`, `insert_entry_if_absent`, compare-and-swap) still put a second type at a held key: a cache now refuses to pull such a redb store with `SchemaConflictError`, where it used to load both records.
|
||||||
|
- A write of the same type under a held key replaces the record by key, and in one `push_all` the last given of several entries under a key and a schema id is the record, whatever their type names. The redb stores are unchanged and still key by type and key.
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name or value find the one with the smallest key. A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`).
|
- `CultCache::foreign_records()` and `ForeignRecord { key, schema_id, schema_name, stored_at }`: the records the last pull found that no registered type claims, sorted by key.
|
||||||
|
- Lookups, matching the C# `GetByName`, `GetByIndex` and `GetGlobal`. `#[cultcache(name)]` and `#[cultcache(index)]` / `#[cultcache(index = "alias")]` on `DatabaseEntry` fields and `#[cultcache(global)]` on the struct give `CultCache::get_by_name`, `get_by_index` and `get_global`. An empty name and a blank index value are not found; records of one type sharing a name find the one with the smallest key (an index value is unique, below). A second record of a global type under another key is refused on every write path and on load with `CultCacheSecondGlobal`. Two `name` members, or two members with one index alias, are refused at compile time. A member's text comes from `CultLookupValue` (`String`, `Option` of one, integers, `bool`, `char`).
|
||||||
- `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::<SchemaConflictError>()`.
|
- `SchemaConflictError`, the typed refusal of a write the catalog cannot describe: records of different types under one schema id, or an arrived schema and a registered one that share an id and disagree on the schema name. A whole-store write (`push_all`) refuses with it, names the schema id, the names involved and a record key, writes nothing and leaves the file as it is. A caller tells it apart with `error.downcast_ref::<SchemaConflictError>()`.
|
||||||
|
|
||||||
## [0.3.1]
|
## [0.3.1]
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,14 @@ CultCache is a domain cache with persistence adapters.
|
||||||
|
|
||||||
- `CultCache` is the query and mutation surface.
|
- `CultCache` is the query and mutation surface.
|
||||||
- Domain structs implement `DatabaseEntry`.
|
- Domain structs implement `DatabaseEntry`.
|
||||||
- Entries are stored behind a `type::key` identity, so multiple entry types
|
- A cache holds one record per key whatever its type. Records are held per
|
||||||
can share a logical key without colliding.
|
`(type, key)`, but a write of a second type under a key the cache holds, or a
|
||||||
|
key a store carries foreign, is refused with `SchemaConflictError` naming the
|
||||||
|
key (no store needed); retyping a key is a delete, then an add. A store
|
||||||
|
record no registered type claims loads: the cache does not serve it, lists it
|
||||||
|
in `foreign_records()`, lays it back byte for byte through every write
|
||||||
|
including `put_prepared_batch`, and refuses a write or delete at its key.
|
||||||
|
Registering a type that claims its schema id picks it up at the next pull.
|
||||||
- Each payload is encoded directly from the known `DatabaseEntry` type into
|
- Each payload is encoded directly from the known `DatabaseEntry` type into
|
||||||
MessagePack bytes.
|
MessagePack bytes.
|
||||||
- Backing stores are adapters, not the public data model.
|
- Backing stores are adapters, not the public data model.
|
||||||
|
|
@ -394,6 +400,10 @@ fn outcome(error: &anyhow::Error) {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Store format and rollback
|
||||||
|
|
||||||
|
A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads and writes a v3 single-file store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-rs from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.)
|
||||||
|
|
||||||
## Near-Term Ergonomic Improvements
|
## Near-Term Ergonomic Improvements
|
||||||
|
|
||||||
1. **Derive macro**
|
1. **Derive macro**
|
||||||
|
|
@ -412,7 +422,9 @@ fn outcome(error: &anyhow::Error) {
|
||||||
|
|
||||||
3. **Name and index lookups**
|
3. **Name and index lookups**
|
||||||
- `cache.get_by_name::<T>("Potion")`
|
- `cache.get_by_name::<T>("Potion")`
|
||||||
- `cache.get_by_index::<T>("faction", "Lucent")`
|
- `cache.get_by_index::<T>("callsign", "Lucent-7")`: `#[cultcache(index)]` is unique. Within one type no two
|
||||||
|
records hold one non-blank value of the alias; a put, batch or load that would make two is refused, naming both
|
||||||
|
keys and the alias, and writes nothing. A grouping (faction, kind, status) is not an index: filter `get_all`.
|
||||||
- generated extractors instead of reflection strings
|
- generated extractors instead of reflection strings
|
||||||
|
|
||||||
4. **Schema/version metadata**
|
4. **Schema/version metadata**
|
||||||
|
|
@ -452,7 +464,7 @@ That is the Rust version of the original trick:
|
||||||
|
|
||||||
- C# refuses to store objects outside the `DatabaseEntry` root type.
|
- C# refuses to store objects outside the `DatabaseEntry` root type.
|
||||||
- Runtime-visible `DatabaseEntry` subclasses get dedicated formatters.
|
- Runtime-visible `DatabaseEntry` subclasses get dedicated formatters.
|
||||||
- Unknown or unregistered types are rejected.
|
- A write of an unregistered type is rejected; a stored record of an unregistered type is carried as foreign, not decoded.
|
||||||
- MessagePack payload bytes deserialize through known concrete formatters, not
|
- MessagePack payload bytes deserialize through known concrete formatters, not
|
||||||
an open-world type loader or generic structured-value bridge.
|
an open-world type loader or generic structured-value bridge.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -69,23 +69,16 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result<proc_macro2::TokenStr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let index_members = slots.iter().filter_map(|slot| slot.index.as_ref().map(|alias| (alias, &slot.ident)));
|
let index_members: Vec<_> = slots.iter().filter_map(|slot| slot.index.as_ref().map(|alias| (alias, &slot.ident))).collect();
|
||||||
let index_pushes = index_members.map(|(alias, member)| {
|
let index_rows = index_members.iter().map(|(alias, member)| {
|
||||||
quote! {
|
quote! {
|
||||||
if let ::std::option::Option::Some(value) = ::cultcache_rs::CultLookupValue::lookup_value(&self.#member) {
|
(#alias, (|record: &Self| ::cultcache_rs::CultLookupValue::lookup_value(&record.#member))
|
||||||
held.push((#alias, value));
|
as fn(&Self) -> ::std::option::Option<::std::string::String>)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let index_pushes: Vec<_> = index_pushes.collect();
|
let index_rows: Vec<_> = index_rows.collect();
|
||||||
let indexes_fn = (!index_pushes.is_empty()).then(|| {
|
let unique_indexes = (!index_rows.is_empty()).then(|| {
|
||||||
quote! {
|
quote! { const UNIQUE_INDEXES: &'static [::cultcache_rs::CultIndexMember<Self>] = &[#(#index_rows),*]; }
|
||||||
fn cult_indexes(&self) -> ::std::vec::Vec<(&'static str, ::std::string::String)> {
|
|
||||||
let mut held = ::std::vec::Vec::new();
|
|
||||||
#(#index_pushes)*
|
|
||||||
held
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(quote! {
|
Ok(quote! {
|
||||||
|
|
@ -93,8 +86,8 @@ fn database_entry_codec(input: DeriveInput) -> syn::Result<proc_macro2::TokenStr
|
||||||
const TYPE: &'static str = #entry_type;
|
const TYPE: &'static str = #entry_type;
|
||||||
const SCHEMA_NAME: &'static str = #schema_name;
|
const SCHEMA_NAME: &'static str = #schema_name;
|
||||||
const GLOBAL: bool = #global;
|
const GLOBAL: bool = #global;
|
||||||
|
#unique_indexes
|
||||||
#name_fn
|
#name_fn
|
||||||
#indexes_fn
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#codec
|
#codec
|
||||||
|
|
@ -443,7 +436,8 @@ struct CultCacheSlot {
|
||||||
bytes: bool,
|
bytes: bool,
|
||||||
/// `#[cultcache(name)]`: the member `get_by_name` finds the record by.
|
/// `#[cultcache(name)]`: the member `get_by_name` finds the record by.
|
||||||
name: bool,
|
name: bool,
|
||||||
/// `#[cultcache(index)]` or `#[cultcache(index = "alias")]`: the alias `get_by_index` finds the record by.
|
/// `#[cultcache(index)]` or `#[cultcache(index = "alias")]`: a unique lookup. Within one type no two records hold one
|
||||||
|
/// non-blank value of the alias; `get_by_index` finds the record by it.
|
||||||
index: Option<String>,
|
index: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -941,15 +935,15 @@ mod tests {
|
||||||
);
|
);
|
||||||
assert!(tokens.contains("constGLOBAL:bool=true"), "{tokens}");
|
assert!(tokens.contains("constGLOBAL:bool=true"), "{tokens}");
|
||||||
assert!(tokens.contains("fncult_name"), "{tokens}");
|
assert!(tokens.contains("fncult_name"), "{tokens}");
|
||||||
assert!(tokens.contains("(\"code\",value)"), "{tokens}");
|
assert!(tokens.contains("(\"code\","), "{tokens}");
|
||||||
assert!(tokens.contains("(\"sku\",value)"), "{tokens}");
|
assert!(tokens.contains("(\"sku\","), "{tokens}");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_record_without_lookups_generates_none_of_them() {
|
fn a_record_without_lookups_generates_none_of_them() {
|
||||||
let tokens = entry_tokens(r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0)] a: String }"#);
|
let tokens = entry_tokens(r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0)] a: String }"#);
|
||||||
assert!(tokens.contains("constGLOBAL:bool=false"), "{tokens}");
|
assert!(tokens.contains("constGLOBAL:bool=false"), "{tokens}");
|
||||||
assert!(!tokens.contains("fncult_name") && !tokens.contains("fncult_indexes"), "{tokens}");
|
assert!(!tokens.contains("fncult_name") && !tokens.contains("UNIQUE_INDEXES"), "{tokens}");
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResolveIndexAlias: a blank alias is the member's name.
|
// ResolveIndexAlias: a blank alias is the member's name.
|
||||||
|
|
@ -958,7 +952,7 @@ mod tests {
|
||||||
let tokens = entry_tokens(
|
let tokens = entry_tokens(
|
||||||
r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = " ")] code: String }"#,
|
r#"#[cultcache(type = "t")] struct S { #[cultcache(key = 0, index = " ")] code: String }"#,
|
||||||
);
|
);
|
||||||
assert!(tokens.contains("(\"code\",value)"), "{tokens}");
|
assert!(tokens.contains("(\"code\","), "{tokens}");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
283
packages/cultcache-rs/tests/key_identity_vector.rs
Normal file
283
packages/cultcache-rs/tests/key_identity_vector.rs
Normal file
|
|
@ -0,0 +1,283 @@
|
||||||
|
//! `tests/vectors/key-identity`, written only by the C# reference: a cache holds one record per key whatever its type, and a
|
||||||
|
//! store holding a record no registered type claims loads, lists it as foreign and keeps its bytes
|
||||||
|
//! (`variants:ruling:cache-refuses-retype`, `write-ownership-by-written-type`, `unheld-removal-refused-at-commit`).
|
||||||
|
//! Every row is walked against this cache: each put row twice, once as `put` and once as a one-entry `put_prepared_batch`.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use cultcache_rs::{
|
||||||
|
CacheBackingStore, CultCache, DatabaseEntry, SchemaConflictError, SingleFileMessagePackBackingStore,
|
||||||
|
};
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
// The ids C# writes in the store catalogs and on every record (the vector's README).
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "sha256:b20703a98f49c253898404ab50ba2c630720f0b8a5e1a41614d5650be22799fe", schema = "tests.ki_alpha")]
|
||||||
|
struct Alpha {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "sha256:dee4b727cf2ee04e61ab635ff0c5f3dca7345a411dc5f1018e583ab16009cf94", schema = "tests.ki_beta")]
|
||||||
|
struct Beta {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "sha256:fbc26e351e00c9e7185e8de171c7486aba707c229c949f0aae6b889288af5f6d", schema = "tests.ki_gamma")]
|
||||||
|
struct Gamma {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vectors() -> PathBuf {
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/key-identity")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `[refusal key or nil, held [type, key, text], file [key, schema name] or nil, foreign keys, kept foreign keys]`.
|
||||||
|
type Expect = (Option<Vec<String>>, Vec<(String, String, String)>, Option<Vec<(String, String)>>, Vec<String>, Vec<String>);
|
||||||
|
type Row = (String, Vec<String>, Vec<Vec<Value>>, Expect);
|
||||||
|
|
||||||
|
fn text(value: &Value) -> &str {
|
||||||
|
value.as_str().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register(cache: &mut CultCache, names: &[String]) {
|
||||||
|
for name in names {
|
||||||
|
match name.as_str() {
|
||||||
|
"alpha" => drop(cache.register_entry_type::<Alpha>().unwrap()),
|
||||||
|
"beta" => drop(cache.register_entry_type::<Beta>().unwrap()),
|
||||||
|
"gamma" => drop(cache.register_entry_type::<Gamma>().unwrap()),
|
||||||
|
other => panic!("unknown type {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn short(entry_type: &str) -> &'static str {
|
||||||
|
match entry_type {
|
||||||
|
t if t == Alpha::TYPE => "alpha",
|
||||||
|
t if t == Beta::TYPE => "beta",
|
||||||
|
t if t == Gamma::TYPE => "gamma",
|
||||||
|
other => panic!("unknown entry type {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn put(cache: &mut CultCache, as_batch: bool, kind: &str, key: &str, body: &str) -> anyhow::Result<()> {
|
||||||
|
macro_rules! write {
|
||||||
|
($t:ident) => {{
|
||||||
|
let value = $t { text: body.to_string() };
|
||||||
|
if as_batch {
|
||||||
|
let (entry, _) = cache.prepare_entry(key, &value)?;
|
||||||
|
cache.put_prepared_batch(vec![entry])
|
||||||
|
} else {
|
||||||
|
cache.put(key, &value).map(drop)
|
||||||
|
}
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
match kind {
|
||||||
|
"alpha" => write!(Alpha),
|
||||||
|
"beta" => write!(Beta),
|
||||||
|
"gamma" => write!(Gamma),
|
||||||
|
other => panic!("unknown type {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete(cache: &mut CultCache, kind: &str, key: &str) -> anyhow::Result<bool> {
|
||||||
|
match kind {
|
||||||
|
"alpha" => cache.delete::<Alpha>(key),
|
||||||
|
"beta" => cache.delete::<Beta>(key),
|
||||||
|
"gamma" => cache.delete::<Gamma>(key),
|
||||||
|
other => panic!("unknown type {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_cache(registered: &[String], path: Option<&Path>) -> anyhow::Result<CultCache> {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
register(&mut cache, registered);
|
||||||
|
if let Some(path) = path {
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?;
|
||||||
|
cache.pull_all_backing_stores()?;
|
||||||
|
}
|
||||||
|
Ok(cache)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The records a store file holds, read by a fresh store: (key, schema name, schema id, stored-at, payload).
|
||||||
|
fn file_records(path: &Path) -> Vec<(String, String, Option<String>, String, Vec<u8>)> {
|
||||||
|
let mut records: Vec<_> = SingleFileMessagePackBackingStore::new(path)
|
||||||
|
.pull_all()
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.map(|entry| (entry.key, entry.r#type, entry.schema_id, entry.stored_at, entry.payload))
|
||||||
|
.collect();
|
||||||
|
records.sort_by(|left, right| left.0.encode_utf16().cmp(right.0.encode_utf16()));
|
||||||
|
records
|
||||||
|
}
|
||||||
|
|
||||||
|
fn walk(row: &Row, as_batch: bool) -> Result<(), String> {
|
||||||
|
let (label, registered, ops, expect) = row;
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let mut path: Option<PathBuf> = None;
|
||||||
|
let mut before = Vec::new();
|
||||||
|
let mut cache: Option<CultCache> = None;
|
||||||
|
let mut refused: Option<anyhow::Error> = None;
|
||||||
|
let mut refused_key: Option<String> = None;
|
||||||
|
let mut shared_key: Option<String> = None;
|
||||||
|
let mut wrote = false;
|
||||||
|
for (index, op) in ops.iter().enumerate() {
|
||||||
|
let last = index == ops.len() - 1;
|
||||||
|
let result: anyhow::Result<()> = match text(&op[0]) {
|
||||||
|
"open" => {
|
||||||
|
drop(cache.take());
|
||||||
|
match &op[1] {
|
||||||
|
Value::String(file) if file == "memory" => open_cache(registered, None).map(|opened| cache = Some(opened)),
|
||||||
|
file => {
|
||||||
|
let fresh = dir.path().join("store.cc");
|
||||||
|
if let Value::String(file) = file {
|
||||||
|
std::fs::copy(vectors().join(file), &fresh).unwrap();
|
||||||
|
}
|
||||||
|
before = if fresh.exists() { file_records(&fresh) } else { Vec::new() };
|
||||||
|
let opened = open_cache(registered, Some(&fresh));
|
||||||
|
path = Some(fresh);
|
||||||
|
opened.map(|opened| cache = Some(opened))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"reopen" => {
|
||||||
|
drop(cache.take());
|
||||||
|
open_cache(registered, path.as_deref()).map(|opened| cache = Some(opened))
|
||||||
|
}
|
||||||
|
"attach" => {
|
||||||
|
let file = text(&op[1]);
|
||||||
|
let attached = dir.path().join(file);
|
||||||
|
std::fs::copy(vectors().join(file), &attached).unwrap();
|
||||||
|
let held: Vec<String> = cache.as_ref().unwrap().snapshot().into_iter().map(|entry| entry.key).collect();
|
||||||
|
let shared: Vec<String> = file_records(&attached).into_iter().map(|record| record.0).filter(|key| held.contains(key)).collect();
|
||||||
|
let home: Vec<String> = op[2].as_array().unwrap().iter().map(|name| short_to_type(text(name)).to_string()).collect();
|
||||||
|
let cache = cache.as_mut().unwrap();
|
||||||
|
let attach = cache
|
||||||
|
.add_backing_store(SingleFileMessagePackBackingStore::new(&attached), home)
|
||||||
|
.and_then(|()| cache.pull_all_backing_stores());
|
||||||
|
if attach.is_err() && last && shared.len() == 1 {
|
||||||
|
refused_key = Some(shared[0].clone());
|
||||||
|
shared_key = refused_key.clone();
|
||||||
|
assert_eq!(std::fs::read(&attached).unwrap(), std::fs::read(vectors().join(file)).unwrap(), "{label}: a refused attach leaves the store as it was");
|
||||||
|
}
|
||||||
|
attach
|
||||||
|
}
|
||||||
|
"put" => {
|
||||||
|
wrote = true;
|
||||||
|
put(cache.as_mut().unwrap(), as_batch, text(&op[1]), text(&op[2]), text(&op[3]))
|
||||||
|
}
|
||||||
|
"delete" => delete(cache.as_mut().unwrap(), text(&op[1]), text(&op[2])).map(drop),
|
||||||
|
other => panic!("{label}: unknown op {other}"),
|
||||||
|
};
|
||||||
|
match result {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(error) if last => refused = Some(error),
|
||||||
|
Err(error) => return Err(format!("{label}: op {index} failed: {error:#}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !wrote && as_batch {
|
||||||
|
return Ok(()); // not a write row: the batch variant is the same walk
|
||||||
|
}
|
||||||
|
let Some(cache) = cache.as_ref() else {
|
||||||
|
return Err(format!("{label} (batch {as_batch}): the cache did not open: {:#}", refused.unwrap()));
|
||||||
|
};
|
||||||
|
let last = ops.last().unwrap();
|
||||||
|
if let Some(error) = &refused {
|
||||||
|
if let Some(conflict) = error.downcast_ref::<SchemaConflictError>() {
|
||||||
|
refused_key = Some(conflict.record_key.clone());
|
||||||
|
let named = if text(&last[0]) == "attach" { shared_key.clone().unwrap() } else { text(&last[2]).to_string() };
|
||||||
|
assert_eq!(named, conflict.record_key, "{label}: the refusal names the key written, removed or shared");
|
||||||
|
}
|
||||||
|
let key = refused_key.clone().unwrap_or_default();
|
||||||
|
let message = format!("{error:#}");
|
||||||
|
let (all, delimited) = occurrences(&message, &key);
|
||||||
|
assert!(!key.is_empty() && delimited > 0, "{label}: the refusal names the key as a token: {message}");
|
||||||
|
if text(&last[0]) == "attach" {
|
||||||
|
assert_eq!(all, delimited, "{label}: the key occurs in the refusal only as a delimited token: {message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut held: Vec<(String, String, String)> = cache
|
||||||
|
.snapshot()
|
||||||
|
.into_iter()
|
||||||
|
.map(|entry| {
|
||||||
|
let body = match short(&entry.r#type) {
|
||||||
|
"alpha" => cache.get::<Alpha>(&entry.key).unwrap().unwrap().text,
|
||||||
|
"beta" => cache.get::<Beta>(&entry.key).unwrap().unwrap().text,
|
||||||
|
_ => cache.get::<Gamma>(&entry.key).unwrap().unwrap().text,
|
||||||
|
};
|
||||||
|
(short(&entry.r#type).to_string(), entry.key, body)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
held.sort_by(|a, b| (a.0.as_str(), a.1.encode_utf16().collect::<Vec<_>>()).cmp(&(b.0.as_str(), b.1.encode_utf16().collect::<Vec<_>>())));
|
||||||
|
let foreign: Vec<String> = cache.foreign_records().into_iter().map(|record| record.key).collect();
|
||||||
|
let (file, kept) = match &path {
|
||||||
|
None => (None, Vec::new()),
|
||||||
|
Some(path) => {
|
||||||
|
let after = file_records(path);
|
||||||
|
let kept = foreign
|
||||||
|
.iter()
|
||||||
|
.filter(|key| {
|
||||||
|
let original = before.iter().find(|record| &record.0 == *key);
|
||||||
|
let now = after.iter().find(|record| &record.0 == *key);
|
||||||
|
original.is_some() && original.map(|r| (&r.2, &r.3, &r.4)) == now.map(|r| (&r.2, &r.3, &r.4))
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
// This writer names a catalog entry by the record's type, which for these types is the id; C# names it by the
|
||||||
|
// schema name. Which type the record at a key is, is the verdict, so the id reads as its schema name.
|
||||||
|
let name = |described: String| match described.as_str() {
|
||||||
|
t if t == Alpha::TYPE => Alpha::SCHEMA_NAME.to_string(),
|
||||||
|
t if t == Beta::TYPE => Beta::SCHEMA_NAME.to_string(),
|
||||||
|
t if t == Gamma::TYPE => Gamma::SCHEMA_NAME.to_string(),
|
||||||
|
_ => described,
|
||||||
|
};
|
||||||
|
(Some(after.into_iter().map(|record| (record.0, name(record.1))).collect::<Vec<_>>()), kept)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let observed: Expect = (refused_key.map(|key| vec![key]), held, file, foreign, kept);
|
||||||
|
if &observed == expect { Ok(()) } else { Err(format!("{label} (batch {as_batch}): committed {expect:?} but this cache answers {observed:?}")) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How often `key` occurs in `message`, and how often as a delimited token (not a piece of a longer word, schema name or id).
|
||||||
|
fn occurrences(message: &str, key: &str) -> (usize, usize) {
|
||||||
|
let word = |c: Option<char>| c.is_some_and(|c| c.is_ascii_alphanumeric() || c == '_');
|
||||||
|
let found: Vec<usize> = message.match_indices(key).map(|(at, _)| at).collect();
|
||||||
|
let delimited = found
|
||||||
|
.iter()
|
||||||
|
.filter(|at| !word(message[..**at].chars().next_back()) && !word(message[**at + key.len()..].chars().next()))
|
||||||
|
.count();
|
||||||
|
(found.len(), delimited)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn short_to_type(name: &str) -> &'static str {
|
||||||
|
match name {
|
||||||
|
"alpha" => Alpha::TYPE,
|
||||||
|
"beta" => Beta::TYPE,
|
||||||
|
"gamma" => Gamma::TYPE,
|
||||||
|
other => panic!("unknown type {other}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn key_identity_rows_hold_their_verdicts() {
|
||||||
|
let bytes = std::fs::read(vectors().join("rows.msgpack")).unwrap();
|
||||||
|
let rows: Vec<Row> = rmp_serde::from_slice(&bytes).unwrap();
|
||||||
|
let mut walked = 0;
|
||||||
|
let mut failed = Vec::new();
|
||||||
|
for row in &rows {
|
||||||
|
let mut ok = true;
|
||||||
|
for as_batch in [false, true] {
|
||||||
|
if let Err(message) = walk(row, as_batch) {
|
||||||
|
failed.push(message);
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walked += usize::from(ok);
|
||||||
|
}
|
||||||
|
assert!(failed.is_empty(), "{}", failed.join("\n"));
|
||||||
|
assert_eq!(walked, 12, "rows walked");
|
||||||
|
}
|
||||||
|
|
@ -11,7 +11,7 @@ use cultcache_rs::{
|
||||||
struct Item {
|
struct Item {
|
||||||
#[cultcache(key = 0, name)]
|
#[cultcache(key = 0, name)]
|
||||||
name: String,
|
name: String,
|
||||||
#[cultcache(key = 1, index)]
|
#[cultcache(key = 1)]
|
||||||
category: String,
|
category: String,
|
||||||
#[cultcache(key = 2, index = "sku")]
|
#[cultcache(key = 2, index = "sku")]
|
||||||
code: Option<String>,
|
code: Option<String>,
|
||||||
|
|
@ -30,7 +30,7 @@ struct Gadget {
|
||||||
#[cultcache(key = 0, name)]
|
#[cultcache(key = 0, name)]
|
||||||
name: String,
|
name: String,
|
||||||
#[cultcache(key = 1, index)]
|
#[cultcache(key = 1, index)]
|
||||||
category: String,
|
serial: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sorts after `Item` and has its exact shape: a lookup that ran past the end of its own type would answer from it.
|
/// Sorts after `Item` and has its exact shape: a lookup that ran past the end of its own type would answer from it.
|
||||||
|
|
@ -39,7 +39,7 @@ struct Gadget {
|
||||||
struct Zed {
|
struct Zed {
|
||||||
#[cultcache(key = 0, name)]
|
#[cultcache(key = 0, name)]
|
||||||
name: String,
|
name: String,
|
||||||
#[cultcache(key = 1, index)]
|
#[cultcache(key = 1)]
|
||||||
category: String,
|
category: String,
|
||||||
#[cultcache(key = 2, index = "sku")]
|
#[cultcache(key = 2, index = "sku")]
|
||||||
code: Option<String>,
|
code: Option<String>,
|
||||||
|
|
@ -48,7 +48,7 @@ struct Zed {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
#[cultcache(type = "settings", global)]
|
#[cultcache(type = "settings", schema = "tests.settings", global)]
|
||||||
struct Settings {
|
struct Settings {
|
||||||
#[cultcache(key = 0)]
|
#[cultcache(key = 0)]
|
||||||
value: String,
|
value: String,
|
||||||
|
|
@ -71,7 +71,7 @@ struct Numbered {
|
||||||
#[cultcache(key = 0, name)]
|
#[cultcache(key = 0, name)]
|
||||||
id: u32,
|
id: u32,
|
||||||
#[cultcache(key = 1, index)]
|
#[cultcache(key = 1, index)]
|
||||||
owner: u64,
|
badge: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cache() -> CultCache {
|
fn cache() -> CultCache {
|
||||||
|
|
@ -110,7 +110,6 @@ fn a_record_is_found_by_name_and_by_index_after_a_reopen() {
|
||||||
let reopened = reopened(&path);
|
let reopened = reopened(&path);
|
||||||
let by_name = reopened.get_by_name::<Item>("Potion").unwrap().unwrap();
|
let by_name = reopened.get_by_name::<Item>("Potion").unwrap().unwrap();
|
||||||
assert_eq!(by_name.value, 50);
|
assert_eq!(by_name.value, 50);
|
||||||
assert_eq!(reopened.get_by_index::<Item>("category", "Consumable").unwrap(), Some(by_name.clone()));
|
|
||||||
assert_eq!(reopened.get_by_index::<Item>("sku", "P-1").unwrap(), Some(by_name));
|
assert_eq!(reopened.get_by_index::<Item>("sku", "P-1").unwrap(), Some(by_name));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -121,31 +120,32 @@ fn a_missing_name_or_value_is_none_and_an_unregistered_type_is_refused() {
|
||||||
|
|
||||||
assert_eq!(cache.get_by_name::<Item>("B").unwrap(), None);
|
assert_eq!(cache.get_by_name::<Item>("B").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_name::<Item>("a").unwrap(), None, "names compare ordinally");
|
assert_eq!(cache.get_by_name::<Item>("a").unwrap(), None, "names compare ordinally");
|
||||||
assert_eq!(cache.get_by_index::<Item>("category", "other").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("sku", "other").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_index::<Item>("nosuchalias", "cat").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("nosuchalias", "a-1").unwrap(), None);
|
||||||
|
assert_eq!(cache.get_by_index::<Item>("category", "cat").unwrap(), None, "category is a grouping, not an index");
|
||||||
assert_eq!(cache.get_by_index::<Item>("code", "a-1").unwrap(), None, "the alias is sku, not the member name");
|
assert_eq!(cache.get_by_index::<Item>("code", "a-1").unwrap(), None, "the alias is sku, not the member name");
|
||||||
|
|
||||||
let bare = CultCache::new();
|
let bare = CultCache::new();
|
||||||
assert!(bare.get_by_name::<Item>("A").is_err());
|
assert!(bare.get_by_name::<Item>("A").is_err());
|
||||||
assert!(bare.get_by_index::<Item>("category", "cat").is_err());
|
assert!(bare.get_by_index::<Item>("sku", "a-1").is_err());
|
||||||
assert!(bare.get_global::<Settings>().is_err());
|
assert!(bare.get_global::<Settings>().is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_index_without_an_alias_is_named_by_its_member() {
|
fn an_index_without_an_alias_is_named_by_its_member() {
|
||||||
let mut cache = cache();
|
let mut cache = cache();
|
||||||
cache.put("item:a", &item("A", "cat", None)).unwrap();
|
cache.put("numbered:1", &Numbered { id: 1, badge: 7 }).unwrap();
|
||||||
assert!(cache.get_by_index::<Item>("category", "cat").unwrap().is_some());
|
assert!(cache.get_by_index::<Numbered>("badge", "7").unwrap().is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
// A blank name is no name; a blank index value is not indexed; a nil member holds no value.
|
// A blank name is no name; a blank index value is not indexed; a nil member holds no value.
|
||||||
#[test]
|
#[test]
|
||||||
fn blank_and_nil_values_are_not_found() {
|
fn blank_and_nil_values_are_not_found() {
|
||||||
let mut cache = cache();
|
let mut cache = cache();
|
||||||
cache.put("item:blank", &item("", " ", None)).unwrap();
|
cache.put("item:blank", &item("", " ", Some(" "))).unwrap();
|
||||||
assert_eq!(cache.get_by_name::<Item>("").unwrap(), None);
|
assert_eq!(cache.get_by_name::<Item>("").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_index::<Item>("category", " ").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("sku", " ").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_index::<Item>("category", "").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("sku", "").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_index::<Item>("sku", "").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("sku", "").unwrap(), None);
|
||||||
|
|
||||||
// Only emptiness voids a name: whitespace is a name, as in C# (Length > 0).
|
// Only emptiness voids a name: whitespace is a name, as in C# (Length > 0).
|
||||||
|
|
@ -156,68 +156,41 @@ fn blank_and_nil_values_are_not_found() {
|
||||||
#[test]
|
#[test]
|
||||||
fn lookups_never_cross_types() {
|
fn lookups_never_cross_types() {
|
||||||
let mut cache = cache();
|
let mut cache = cache();
|
||||||
cache.put("item:a", &item("same", "same", None)).unwrap();
|
cache.put("item:a", &item("same", "same", Some("same"))).unwrap();
|
||||||
cache.put("gadget:a", &Gadget { name: "same".into(), category: "same".into() }).unwrap();
|
cache.put("gadget:a", &Gadget { name: "same".into(), serial: "same".into() }).unwrap();
|
||||||
cache.put("zed:a", &Zed { name: "same".into(), category: "same".into(), code: None, value: 0 }).unwrap();
|
cache.put("zed:a", &Zed { name: "same".into(), category: "same".into(), code: None, value: 0 }).unwrap();
|
||||||
|
|
||||||
assert_eq!(cache.get_by_name::<Gadget>("same").unwrap().unwrap().name, "same");
|
assert_eq!(cache.get_by_name::<Gadget>("same").unwrap().unwrap().name, "same");
|
||||||
cache.delete::<Item>("item:a").unwrap();
|
cache.delete::<Item>("item:a").unwrap();
|
||||||
assert_eq!(cache.get_by_name::<Item>("same").unwrap(), None);
|
assert_eq!(cache.get_by_name::<Item>("same").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_index::<Item>("category", "same").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("sku", "same").unwrap(), None);
|
||||||
assert!(cache.get_by_name::<Gadget>("same").unwrap().is_some());
|
assert!(cache.get_by_name::<Gadget>("same").unwrap().is_some());
|
||||||
assert!(cache.get_by_index::<Gadget>("category", "same").unwrap().is_some());
|
assert!(cache.get_by_index::<Gadget>("serial", "same").unwrap().is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_name_and_index_follow_a_rewritten_record_and_leave_a_removed_one() {
|
fn a_name_and_index_follow_a_rewritten_record_and_leave_a_removed_one() {
|
||||||
let mut cache = cache();
|
let mut cache = cache();
|
||||||
cache.put("item:a", &item("old", "c1", None)).unwrap();
|
cache.put("item:a", &item("old", "cat", Some("c1"))).unwrap();
|
||||||
cache.put("item:a", &item("new", "c2", None)).unwrap();
|
cache.put("item:a", &item("new", "cat", Some("c2"))).unwrap();
|
||||||
|
|
||||||
assert_eq!(cache.get_by_name::<Item>("old").unwrap(), None, "a stale name is not found");
|
assert_eq!(cache.get_by_name::<Item>("old").unwrap(), None, "a stale name is not found");
|
||||||
assert_eq!(cache.get_by_index::<Item>("category", "c1").unwrap(), None, "a stale value is not found");
|
assert_eq!(cache.get_by_index::<Item>("sku", "c1").unwrap(), None, "a stale value is not found");
|
||||||
assert!(cache.get_by_name::<Item>("new").unwrap().is_some());
|
assert!(cache.get_by_name::<Item>("new").unwrap().is_some());
|
||||||
assert!(cache.get_by_index::<Item>("category", "c2").unwrap().is_some());
|
assert!(cache.get_by_index::<Item>("sku", "c2").unwrap().is_some());
|
||||||
|
|
||||||
cache.delete::<Item>("item:a").unwrap();
|
cache.delete::<Item>("item:a").unwrap();
|
||||||
assert_eq!(cache.get_by_name::<Item>("new").unwrap(), None);
|
assert_eq!(cache.get_by_name::<Item>("new").unwrap(), None);
|
||||||
assert_eq!(cache.get_by_index::<Item>("category", "c2").unwrap(), None);
|
assert_eq!(cache.get_by_index::<Item>("sku", "c2").unwrap(), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn non_string_members_are_found_by_their_text() {
|
fn non_string_members_are_found_by_their_text() {
|
||||||
let mut cache = cache();
|
let mut cache = cache();
|
||||||
let owner = 161_u64;
|
let badge = 161_u64;
|
||||||
cache.put("numbered:1", &Numbered { id: 42, owner }).unwrap();
|
cache.put("numbered:1", &Numbered { id: 42, badge }).unwrap();
|
||||||
assert!(cache.get_by_name::<Numbered>("42").unwrap().is_some());
|
assert!(cache.get_by_name::<Numbered>("42").unwrap().is_some());
|
||||||
assert!(cache.get_by_index::<Numbered>("owner", "161").unwrap().is_some());
|
assert!(cache.get_by_index::<Numbered>("badge", "161").unwrap().is_some());
|
||||||
}
|
|
||||||
|
|
||||||
// VariantTests.RemovingTheHolderADuplicateValueFindsLeavesTheOtherFindable: records sharing a value find the one with
|
|
||||||
// the smallest key, whatever the write order, live and after a reopen; removing the holder finds the other.
|
|
||||||
#[test]
|
|
||||||
fn records_sharing_a_value_find_the_smallest_key_and_removing_it_finds_the_next() {
|
|
||||||
let dir = tempfile::tempdir().unwrap();
|
|
||||||
let path = dir.path().join("shared.cc");
|
|
||||||
{
|
|
||||||
let mut cache = stored_cache(&path);
|
|
||||||
cache.put("item:p2", &item("p2", "x", None)).unwrap();
|
|
||||||
cache.put("item:p1", &item("p1", "x", None)).unwrap();
|
|
||||||
let found = |cache: &CultCache| cache.get_by_index::<Item>("category", "x").unwrap().map(|found| found.name);
|
|
||||||
assert_eq!(found(&cache).as_deref(), Some("p1"), "the smallest key, though written second");
|
|
||||||
assert_eq!(reopened(&path).get_by_index::<Item>("category", "x").unwrap().unwrap().name, "p1");
|
|
||||||
|
|
||||||
cache.delete::<Item>("item:p1").unwrap();
|
|
||||||
assert_eq!(found(&cache).as_deref(), Some("p2"), "removing the holder finds the other one");
|
|
||||||
|
|
||||||
cache.put("item:p1", &item("p1", "x", None)).unwrap();
|
|
||||||
assert_eq!(found(&cache).as_deref(), Some("p1"));
|
|
||||||
cache.delete::<Item>("item:p2").unwrap();
|
|
||||||
assert_eq!(found(&cache).as_deref(), Some("p1"), "removing the other changes nothing");
|
|
||||||
cache.delete::<Item>("item:p1").unwrap();
|
|
||||||
assert_eq!(found(&cache), None);
|
|
||||||
}
|
|
||||||
assert_eq!(reopened(&path).get_by_index::<Item>("category", "x").unwrap(), None);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -244,6 +217,71 @@ fn a_global_is_none_until_written_and_survives_a_reopen() {
|
||||||
assert_eq!(reopened(&path).get_global::<Settings>().unwrap(), Some(settings("durable authority")));
|
assert_eq!(reopened(&path).get_global::<Settings>().unwrap(), Some(settings("durable authority")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `Settings` as an ordinary type: the same type id and schema, no global rule.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "settings", schema = "tests.settings")]
|
||||||
|
struct LooseSettings {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
value: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Declaring a held type global judges the records it holds: two are already a second, so the registration is
|
||||||
|
// refused and the type stays ordinary.
|
||||||
|
#[test]
|
||||||
|
fn registering_a_type_global_over_two_held_records_is_refused_and_changes_nothing() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<LooseSettings>().unwrap();
|
||||||
|
cache.put("a", &LooseSettings { value: "one".into() }).unwrap();
|
||||||
|
cache.put("b", &LooseSettings { value: "two".into() }).unwrap();
|
||||||
|
|
||||||
|
let refusal = cache.register_entry_type::<Settings>().unwrap_err();
|
||||||
|
assert!(refusal.downcast_ref::<CultCacheSecondGlobal>().is_some(), "{refusal:#}");
|
||||||
|
|
||||||
|
cache.put("c", &LooseSettings { value: "three".into() }).unwrap();
|
||||||
|
assert_eq!(cache.get_all_with_keys::<LooseSettings>().unwrap().len(), 3, "still ordinary");
|
||||||
|
}
|
||||||
|
|
||||||
|
// One held record may become the global; a second is then refused as a second global.
|
||||||
|
#[test]
|
||||||
|
fn registering_a_type_global_over_one_held_record_is_admitted() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<LooseSettings>().unwrap();
|
||||||
|
cache.put("a", &LooseSettings { value: "one".into() }).unwrap();
|
||||||
|
cache.register_entry_type::<Settings>().unwrap();
|
||||||
|
let refusal = cache.put("b", &Settings { value: "two".into() }).unwrap_err();
|
||||||
|
assert!(refusal.downcast_ref::<CultCacheSecondGlobal>().is_some(), "{refusal:#}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A global with an indexed member, and the same type and schema declaring that member as an integer.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "indexed_global", schema = "tests.indexed_global", global)]
|
||||||
|
struct IndexedGlobal {
|
||||||
|
#[cultcache(key = 0, index = "V")]
|
||||||
|
v: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "indexed_global", schema = "tests.indexed_global", global)]
|
||||||
|
struct RetypedGlobal {
|
||||||
|
#[cultcache(key = 0, index = "V")]
|
||||||
|
v: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
// A refused re-registration over an already-global type leaves it global: the rule keeps refusing a second record.
|
||||||
|
#[test]
|
||||||
|
fn a_refused_reregistration_leaves_a_global_type_global() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<IndexedGlobal>().unwrap();
|
||||||
|
cache.put("a", &IndexedGlobal { v: "x".into() }).unwrap();
|
||||||
|
|
||||||
|
assert!(cache.register_entry_type::<RetypedGlobal>().is_err(), "the held text cannot be decoded as an integer");
|
||||||
|
|
||||||
|
let refusal = cache.put("b", &IndexedGlobal { v: "y".into() }).unwrap_err();
|
||||||
|
assert!(refusal.downcast_ref::<CultCacheSecondGlobal>().is_some(), "{refusal:#}");
|
||||||
|
assert_eq!(cache.get_global::<IndexedGlobal>().unwrap(), Some(IndexedGlobal { v: "x".into() }));
|
||||||
|
assert_eq!(cache.get_by_index::<IndexedGlobal>("V", "x").unwrap(), cache.get::<IndexedGlobal>("a").unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_type_that_is_not_a_global_has_no_global() {
|
fn a_type_that_is_not_a_global_has_no_global() {
|
||||||
let mut cache = cache();
|
let mut cache = cache();
|
||||||
|
|
|
||||||
775
packages/cultcache-rs/tests/unique_index.rs
Normal file
775
packages/cultcache-rs/tests/unique_index.rs
Normal file
|
|
@ -0,0 +1,775 @@
|
||||||
|
//! `#[cultcache(index)]` is unique (`variants:ruling:unique-indexes`; C# `RefuseIndexSharing`): within one type and one
|
||||||
|
//! alias no two records hold one non-blank value, judged on the result of each admission, on every write path and every
|
||||||
|
//! load. The refusal text is informational; the contract is the verdict, so most pins assert the verdict and the
|
||||||
|
//! shared vectors in `tests/vectors/unique-index` pin the text and the canonical index text.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use cultcache_rs::{
|
||||||
|
CultCache, CultCacheEnvelope, CultCacheSoaTable, DatabaseEntry, SingleFileMessagePackBackingStore,
|
||||||
|
};
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
/// The vectors' `tests.unique_item`: Sku is unique, Category a grouping.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "unique_item", schema = "tests.unique_item")]
|
||||||
|
struct Unique {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1)]
|
||||||
|
category: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same type with no index: a role, which writes the duplicates `Unique` refuses.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "unique_item", schema = "tests.unique_item")]
|
||||||
|
struct Twin {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1)]
|
||||||
|
category: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Two unique indexes declared out of alphabetical order, so no ordering is an accident of declaration.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "multi", schema = "Multi")]
|
||||||
|
struct Multi {
|
||||||
|
#[cultcache(key = 0, index = "B")]
|
||||||
|
b: Option<String>,
|
||||||
|
#[cultcache(key = 1, index = "A")]
|
||||||
|
a: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Another type sharing the alias `Sku`: types never conflict.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "other", schema = "Other")]
|
||||||
|
struct Other {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "plain", schema = "Plain")]
|
||||||
|
struct Plain {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
value: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "vs", schema = "VS")]
|
||||||
|
struct VString {
|
||||||
|
#[cultcache(key = 0, index = "V")]
|
||||||
|
v: String,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "vi", schema = "VI")]
|
||||||
|
struct VInt {
|
||||||
|
#[cultcache(key = 0, index = "V")]
|
||||||
|
v: i64,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "vu", schema = "VU")]
|
||||||
|
struct VUint {
|
||||||
|
#[cultcache(key = 0, index = "V")]
|
||||||
|
v: u64,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "vb", schema = "VB")]
|
||||||
|
struct VBool {
|
||||||
|
#[cultcache(key = 0, index = "V")]
|
||||||
|
v: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The refusals vector's other schemas: `Sku` is unique in each; `Alt` is a second unique index in `uq_pair` (alias
|
||||||
|
/// `Alt`) and `uq_alias` (alias `alt`, sorting after `Sku` only under an ordinal compare); alpha and beta are plain.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "uq_pair", schema = "tests.uq_pair")]
|
||||||
|
struct UqPair {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1, index = "Alt")]
|
||||||
|
alt: String,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "uq_alias", schema = "tests.uq_alias")]
|
||||||
|
struct UqAlias {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1, index = "alt")]
|
||||||
|
alt: String,
|
||||||
|
}
|
||||||
|
/// Aliases U+FF21 and U+10400: UTF-16 puts the astral alias first (D801 DC00 < FF21), code-point order the other way.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "uq_wide", schema = "tests.uq_wide")]
|
||||||
|
struct UqWide {
|
||||||
|
#[cultcache(key = 0, index = "\u{FF21}")]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1, index = "\u{10400}")]
|
||||||
|
alt: String,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "uq_alpha", schema = "tests.uq_alpha")]
|
||||||
|
struct UqAlpha {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1)]
|
||||||
|
second: String,
|
||||||
|
}
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "uq_beta", schema = "tests.uq_beta")]
|
||||||
|
struct UqBeta {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1)]
|
||||||
|
second: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique(sku: &str) -> Unique {
|
||||||
|
Unique { sku: sku.into(), category: "c".into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn multi(a: Option<&str>, b: Option<&str>) -> Multi {
|
||||||
|
Multi { a: a.map(Into::into), b: b.map(Into::into) }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn refusal(k1: &str, k2: &str, schema: &str, alias: &str) -> String {
|
||||||
|
format!("Records {k1} and {k2} ({schema}) both hold unique index '{alias}'; an index value names one record.")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cache_at(path: &Path) -> CultCache {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Unique>().unwrap();
|
||||||
|
cache.register_entry_type::<Multi>().unwrap();
|
||||||
|
cache.register_entry_type::<Other>().unwrap();
|
||||||
|
cache.register_entry_type::<Plain>().unwrap();
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
cache
|
||||||
|
}
|
||||||
|
|
||||||
|
fn memory_cache() -> CultCache {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Unique>().unwrap();
|
||||||
|
cache.register_entry_type::<Multi>().unwrap();
|
||||||
|
cache.register_entry_type::<Other>().unwrap();
|
||||||
|
cache
|
||||||
|
}
|
||||||
|
|
||||||
|
fn twin_cache_at(path: &Path) -> CultCache {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Twin>().unwrap();
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
cache
|
||||||
|
}
|
||||||
|
|
||||||
|
fn error_text(result: anyhow::Result<impl std::fmt::Debug>) -> String {
|
||||||
|
format!("{:#}", result.unwrap_err())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn skus(cache: &CultCache) -> Vec<(String, String)> {
|
||||||
|
cache.get_all_with_keys::<Unique>().unwrap().into_iter().map(|(key, value)| (key, value.sku)).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_derive_emits_the_unique_aliases() {
|
||||||
|
fn aliases<T: DatabaseEntry>() -> Vec<&'static str> {
|
||||||
|
T::UNIQUE_INDEXES.iter().map(|(alias, _)| *alias).collect()
|
||||||
|
}
|
||||||
|
assert_eq!(aliases::<Unique>(), ["Sku"]);
|
||||||
|
assert_eq!(aliases::<Multi>(), ["B", "A"]);
|
||||||
|
assert!(aliases::<Plain>().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_put_of_a_held_value_at_another_key_is_refused_and_changes_nothing() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("u.cc");
|
||||||
|
let mut cache = cache_at(&path);
|
||||||
|
cache.put("item:b", &unique("S-1")).unwrap();
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
|
||||||
|
let text = error_text(cache.put("item:a", &unique("S-1")));
|
||||||
|
assert_eq!(text, refusal("item:a", "item:b", "tests.unique_item", "Sku"));
|
||||||
|
assert!(!text.contains("S-1"), "the refusal never names the value");
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), bytes, "the store is untouched");
|
||||||
|
assert_eq!(cache.get::<Unique>("item:a").unwrap(), None);
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S-1").unwrap().unwrap().sku, "S-1");
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S-1").unwrap(), cache.get::<Unique>("item:b").unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
// The old bug: write b then a sharing a value used to answer whichever key sorted first.
|
||||||
|
#[test]
|
||||||
|
fn the_first_holder_keeps_the_value_whatever_the_key_order() {
|
||||||
|
let mut cache = memory_cache();
|
||||||
|
cache.put("item:z", &unique("S")).unwrap();
|
||||||
|
assert!(cache.put("item:a", &unique("S")).is_err());
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap().unwrap(), unique("S"));
|
||||||
|
assert_eq!(skus(&cache), vec![("item:z".to_string(), "S".to_string())]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_record_rewriting_its_own_key_keeps_its_value_and_a_change_releases_the_old_one() {
|
||||||
|
let mut cache = memory_cache();
|
||||||
|
cache.put("k", &unique("S")).unwrap();
|
||||||
|
cache.put("k", &Unique { sku: "S".into(), category: "other".into() }).unwrap();
|
||||||
|
assert!(cache.get_by_index::<Unique>("Sku", "S").unwrap().is_some());
|
||||||
|
|
||||||
|
cache.put("k", &unique("T")).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), None, "a stale value is not found");
|
||||||
|
cache.put("j", &unique("S")).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap().unwrap().sku, "S");
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "T").unwrap().unwrap().sku, "T");
|
||||||
|
// Updating through `update` is the same rule.
|
||||||
|
assert!(cache.update::<Unique, _>("k", |_| unique("S")).is_err());
|
||||||
|
assert_eq!(cache.get::<Unique>("k").unwrap().unwrap().sku, "T");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn delete_releases_a_value() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("d.cc");
|
||||||
|
let mut cache = cache_at(&path);
|
||||||
|
cache.put("item:a", &unique("S")).unwrap();
|
||||||
|
assert!(cache.put("item:b", &unique("S")).is_err());
|
||||||
|
assert!(cache.delete::<Unique>("item:a").unwrap());
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), None);
|
||||||
|
cache.put("item:b", &unique("S")).unwrap();
|
||||||
|
assert!(cache.get_by_index::<Unique>("Sku", "S").unwrap().is_some());
|
||||||
|
assert!(!cache.delete::<Unique>("item:a").unwrap(), "deleting an absent key releases nothing");
|
||||||
|
assert!(cache.get_by_index::<Unique>("Sku", "S").unwrap().is_some());
|
||||||
|
assert!(cache_at(&path).get_by_index::<Unique>("Sku", "S").unwrap().is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn types_sharing_an_alias_and_a_value_do_not_conflict() {
|
||||||
|
let mut cache = memory_cache();
|
||||||
|
cache.put("a", &unique("S")).unwrap();
|
||||||
|
cache.put("oa", &Other { sku: "S".into() }).unwrap();
|
||||||
|
cache.put("ob", &Other { sku: "T".into() }).unwrap();
|
||||||
|
assert!(cache.put("ob", &Other { sku: "S".into() }).is_err());
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap().unwrap(), unique("S"));
|
||||||
|
assert_eq!(cache.get_by_index::<Other>("Sku", "S").unwrap().unwrap(), Other { sku: "S".into() });
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_alias_the_type_does_not_declare_finds_nothing() {
|
||||||
|
let mut cache = memory_cache();
|
||||||
|
cache.put("a", &unique("S")).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("sku", "S").unwrap(), None);
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Category", "c").unwrap(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blank_and_none_values_never_conflict() {
|
||||||
|
let mut cache = memory_cache();
|
||||||
|
for (key, sku) in [("a", " "), ("b", ""), ("c", "\u{85}"), ("d", "\t\n\u{a0}\u{3000}")] {
|
||||||
|
cache.put(key, &unique(sku)).unwrap();
|
||||||
|
}
|
||||||
|
cache.put("m1", &multi(None, None)).unwrap();
|
||||||
|
cache.put("m2", &multi(None, Some(" "))).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", " ").unwrap(), None);
|
||||||
|
// U+FEFF, U+001F, U+200B, U+180E are values, never trimmed.
|
||||||
|
cache.put("e", &unique("\u{feff}")).unwrap();
|
||||||
|
assert!(cache.put("f", &unique("\u{feff}")).is_err());
|
||||||
|
cache.put("g", &unique(" S ")).unwrap();
|
||||||
|
cache.put("h", &unique("S")).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", " S ").unwrap().unwrap().sku, " S ");
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap().unwrap().sku, "S");
|
||||||
|
}
|
||||||
|
|
||||||
|
// C# `string.IsNullOrWhiteSpace` over every code unit: exactly these 25 are blank (probed at 7760219b, dotnet 10).
|
||||||
|
#[test]
|
||||||
|
fn the_blank_set_is_exactly_the_csharp_white_space() {
|
||||||
|
let blank: Vec<u32> = (0x9..=0xD)
|
||||||
|
.chain([0x20, 0x85, 0xA0, 0x1680])
|
||||||
|
.chain(0x2000..=0x200A)
|
||||||
|
.chain([0x2028, 0x2029, 0x202F, 0x205F, 0x3000])
|
||||||
|
.collect();
|
||||||
|
assert_eq!(blank.len(), 25);
|
||||||
|
let mut cache = memory_cache();
|
||||||
|
let mut found = Vec::new();
|
||||||
|
for unit in (0..=0xFFFF_u32).filter(|unit| char::from_u32(*unit).is_some()) {
|
||||||
|
let text = char::from_u32(unit).unwrap().to_string();
|
||||||
|
cache.put("a", &unique(&text)).unwrap();
|
||||||
|
let second = cache.put("b", &unique(&text));
|
||||||
|
if second.is_ok() {
|
||||||
|
found.push(unit);
|
||||||
|
cache.delete::<Unique>("b").unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", &text).unwrap(), None, "U+{unit:04X} blank is never found");
|
||||||
|
} else {
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", &text).unwrap().unwrap().sku, text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(found, blank);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn staged<T: DatabaseEntry>(cache: &CultCache, rows: &[(&str, T)]) -> Vec<CultCacheEnvelope> {
|
||||||
|
rows.iter().map(|(key, value)| cache.prepare_entry(*key, value).unwrap().0).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two unique indexes, collision on the second-declared, through every write path.
|
||||||
|
#[test]
|
||||||
|
fn every_write_path_refuses_and_changes_nothing() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("paths.cc");
|
||||||
|
let mut cache = cache_at(&path);
|
||||||
|
cache.put("m:held", &multi(Some("a1"), Some("b1"))).unwrap();
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
let before = cache.snapshot();
|
||||||
|
|
||||||
|
let clash_b = multi(Some("a2"), Some("b1"));
|
||||||
|
let clash_a = multi(Some("a1"), Some("b2"));
|
||||||
|
for (clash, alias) in [(&clash_b, "B"), (&clash_a, "A")] {
|
||||||
|
let pick = |text: &str| {
|
||||||
|
let (first, second) = if "m:new" < "m:held" { ("m:new", "m:held") } else { ("m:held", "m:new") };
|
||||||
|
assert_eq!(text, refusal(first, second, "Multi", alias));
|
||||||
|
};
|
||||||
|
pick(&error_text(cache.put("m:new", clash)));
|
||||||
|
pick(&error_text(cache.update::<Multi, _>("m:new", |_| clash.clone())));
|
||||||
|
let envelope = staged(&cache, &[("m:new", clash.clone())]).remove(0);
|
||||||
|
pick(&error_text(cache.put_envelope::<Multi>(envelope.clone())));
|
||||||
|
pick(&error_text(cache.put_raw_envelope(envelope.clone())));
|
||||||
|
pick(&error_text(cache.put_prepared_batch(vec![envelope])));
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), bytes);
|
||||||
|
assert_eq!(cache.snapshot(), before);
|
||||||
|
assert_eq!(cache.get_by_index::<Multi>("B", "b2").unwrap(), None);
|
||||||
|
}
|
||||||
|
// A clean row through every path lands and is found by both aliases.
|
||||||
|
let fine = multi(Some("a9"), Some("b9"));
|
||||||
|
cache.put("m:1", &fine).unwrap();
|
||||||
|
let envelope = staged(&cache, &[("m:2", multi(Some("a8"), Some("b8")))]).remove(0);
|
||||||
|
cache.put_envelope::<Multi>(envelope).unwrap();
|
||||||
|
let envelope = staged(&cache, &[("m:3", multi(Some("a7"), Some("b7")))]).remove(0);
|
||||||
|
cache.put_raw_envelope(envelope).unwrap();
|
||||||
|
cache.put_prepared_batch(staged(&cache, &[("m:4", multi(Some("a6"), Some("b6")))])).unwrap();
|
||||||
|
for (alias, value, key) in [("A", "a9", 1), ("B", "b8", 2), ("A", "a7", 3), ("B", "b6", 4)] {
|
||||||
|
let found = cache.get_by_index::<Multi>(alias, value).unwrap().unwrap();
|
||||||
|
assert_eq!(Some(found), cache.get::<Multi>(&format!("m:{key}")).unwrap());
|
||||||
|
}
|
||||||
|
// Held values survive a reopen of the store: the load installs the same holders.
|
||||||
|
let mut reopened = cache_at(&path);
|
||||||
|
assert!(reopened.put("m:x", &multi(None, Some("b9"))).is_err());
|
||||||
|
assert!(reopened.put("m:x", &multi(Some("a6"), None)).is_err());
|
||||||
|
assert!(reopened.put("m:x", &multi(Some("zz"), Some("yy"))).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_batch_is_judged_whole() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("batch.cc");
|
||||||
|
let mut cache = cache_at(&path);
|
||||||
|
cache.put("a", &unique("X")).unwrap();
|
||||||
|
cache.put("b", &unique("Y")).unwrap();
|
||||||
|
|
||||||
|
// A swap inside one batch lands.
|
||||||
|
cache.put_prepared_batch(staged(&cache, &[("a", unique("Y")), ("b", unique("X"))])).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "Y").unwrap(), cache.get::<Unique>("a").unwrap());
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "X").unwrap(), cache.get::<Unique>("b").unwrap());
|
||||||
|
assert_eq!(cache_at(&path).get_by_index::<Unique>("Sku", "X").unwrap().unwrap().sku, "X");
|
||||||
|
|
||||||
|
// A batch taking a value a record being replaced just released lands; one holding a value twice is refused whole.
|
||||||
|
cache.put_prepared_batch(staged(&cache, &[("a", unique("Z")), ("c", unique("Y"))])).unwrap();
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
let before = cache.snapshot();
|
||||||
|
let text = error_text(cache.put_prepared_batch(staged(&cache, &[("p", unique("N")), ("q", unique("N")), ("r", unique("W"))])));
|
||||||
|
assert_eq!(text, refusal("p", "q", "tests.unique_item", "Sku"));
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), bytes);
|
||||||
|
assert_eq!(cache.snapshot(), before);
|
||||||
|
|
||||||
|
// A key staged twice lands as its last envelope.
|
||||||
|
cache.put_prepared_batch(staged(&cache, &[("d", unique("D1")), ("d", unique("D2")), ("e", unique("D1"))])).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "D2").unwrap(), cache.get::<Unique>("d").unwrap());
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "D1").unwrap(), cache.get::<Unique>("e").unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_load_of_a_duplicate_image_is_refused_and_keeps_the_prior_view() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("load.cc");
|
||||||
|
let mut cache = cache_at(&path);
|
||||||
|
cache.put("item:keep", &unique("K")).unwrap();
|
||||||
|
|
||||||
|
let mut twin = twin_cache_at(&path);
|
||||||
|
twin.put("item:b", &Twin { sku: "S".into(), category: "c".into() }).unwrap();
|
||||||
|
twin.put("item:a", &Twin { sku: "S".into(), category: "c".into() }).unwrap();
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
|
||||||
|
let before = cache.snapshot();
|
||||||
|
assert_eq!(error_text(cache.pull_all_backing_stores()), refusal("item:a", "item:b", "tests.unique_item", "Sku"));
|
||||||
|
assert_eq!(cache.snapshot(), before, "a refused pull admits nothing");
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "K").unwrap().unwrap().sku, "K");
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), bytes);
|
||||||
|
|
||||||
|
// load_soa of the same image through a cache with no store.
|
||||||
|
let table: CultCacheSoaTable = twin.snapshot_soa();
|
||||||
|
let mut soa = memory_cache();
|
||||||
|
soa.put("item:keep", &unique("K")).unwrap();
|
||||||
|
let before = soa.snapshot();
|
||||||
|
assert_eq!(error_text(soa.load_soa(table)), refusal("item:a", "item:b", "tests.unique_item", "Sku"));
|
||||||
|
assert_eq!(soa.snapshot(), before);
|
||||||
|
assert!(soa.get_by_index::<Unique>("Sku", "K").unwrap().is_some());
|
||||||
|
|
||||||
|
// A clean image loads and replaces the holders with the image's.
|
||||||
|
let mut clean = twin_cache_at(&dir.path().join("clean.cc"));
|
||||||
|
clean.put("item:x", &Twin { sku: "X".into(), category: "c".into() }).unwrap();
|
||||||
|
soa.load_soa(clean.snapshot_soa()).unwrap();
|
||||||
|
assert_eq!(soa.get_by_index::<Unique>("Sku", "K").unwrap(), None);
|
||||||
|
assert!(soa.get_by_index::<Unique>("Sku", "X").unwrap().is_some());
|
||||||
|
soa.put("item:y", &unique("K")).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_multi_index_store_with_a_duplicate_on_the_second_alias_refuses_to_open() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("multi.cc");
|
||||||
|
{
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "multi", schema = "Multi")]
|
||||||
|
struct Role {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
b: Option<String>,
|
||||||
|
#[cultcache(key = 1)]
|
||||||
|
a: Option<String>,
|
||||||
|
}
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Role>().unwrap();
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
cache.put("m:1", &Role { b: Some("b".into()), a: Some("a1".into()) }).unwrap();
|
||||||
|
cache.put("m:2", &Role { b: Some("b".into()), a: Some("a2".into()) }).unwrap();
|
||||||
|
}
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Multi>().unwrap();
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
assert_eq!(error_text(cache.pull_all_backing_stores()), refusal("m:1", "m:2", "Multi", "B"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_bool_index_is_found_by_the_csharp_text() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<VBool>().unwrap();
|
||||||
|
cache.put("t", &VBool { v: true }).unwrap();
|
||||||
|
assert!(cache.put("u", &VBool { v: true }).is_err());
|
||||||
|
cache.put("f", &VBool { v: false }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VBool>("V", "True").unwrap().unwrap().v, true);
|
||||||
|
assert_eq!(cache.get_by_index::<VBool>("V", "False").unwrap().unwrap().v, false);
|
||||||
|
assert_eq!(cache.get_by_index::<VBool>("V", "true").unwrap(), None);
|
||||||
|
assert_eq!(cache.get_by_index::<VBool>("V", "false").unwrap(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- shared vectors, written by the C# reference ----
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum Cell {
|
||||||
|
Nil,
|
||||||
|
Bool(bool),
|
||||||
|
Int(i64),
|
||||||
|
Uint(u64),
|
||||||
|
Double(f64),
|
||||||
|
Text(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vectors() -> PathBuf {
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/unique-index")
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValueRow = (String, String, Cell, Option<String>);
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vectors_values_give_the_csharp_index_text() {
|
||||||
|
let bytes = std::fs::read(vectors().join("values.msgpack")).unwrap();
|
||||||
|
let rows: Vec<ValueRow> = rmp_serde::from_slice(&bytes).unwrap();
|
||||||
|
assert!(!rows.is_empty());
|
||||||
|
let mut checked = 0;
|
||||||
|
for (label, kind, value, text) in rows {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<VString>().unwrap();
|
||||||
|
cache.register_entry_type::<VInt>().unwrap();
|
||||||
|
cache.register_entry_type::<VUint>().unwrap();
|
||||||
|
cache.register_entry_type::<VBool>().unwrap();
|
||||||
|
// `double` rows are skipped: CultLookupValue has no float impl, so Rust cannot index a double member.
|
||||||
|
match (kind.as_str(), value) {
|
||||||
|
("string", Cell::Text(held)) => {
|
||||||
|
cache.put("one", &VString { v: held.clone() }).unwrap();
|
||||||
|
match text {
|
||||||
|
Some(text) => {
|
||||||
|
assert_eq!(cache.get_by_index::<VString>("V", &text).unwrap().unwrap().v, held, "{label}");
|
||||||
|
assert!(cache.put("two", &VString { v: held }).is_err(), "{label}");
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
cache.put("two", &VString { v: held.clone() }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VString>("V", &held).unwrap(), None, "{label}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
("int64", Cell::Int(held)) => {
|
||||||
|
cache.put("one", &VInt { v: held }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VInt>("V", &text.unwrap()).unwrap().unwrap().v, held, "{label}");
|
||||||
|
assert!(cache.put("two", &VInt { v: held }).is_err(), "{label}");
|
||||||
|
}
|
||||||
|
("uint64", Cell::Uint(held)) => {
|
||||||
|
cache.put("one", &VUint { v: held }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VUint>("V", &text.unwrap()).unwrap().unwrap().v, held, "{label}");
|
||||||
|
assert!(cache.put("two", &VUint { v: held }).is_err(), "{label}");
|
||||||
|
}
|
||||||
|
("uint64", Cell::Int(held)) => {
|
||||||
|
cache.put("one", &VUint { v: held as u64 }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VUint>("V", &text.unwrap()).unwrap().unwrap().v, held as u64, "{label}");
|
||||||
|
}
|
||||||
|
("int64", Cell::Uint(held)) => {
|
||||||
|
cache.put("one", &VInt { v: held as i64 }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VInt>("V", &text.unwrap()).unwrap().unwrap().v, held as i64, "{label}");
|
||||||
|
}
|
||||||
|
("bool", Cell::Bool(held)) => {
|
||||||
|
cache.put("one", &VBool { v: held }).unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<VBool>("V", &text.unwrap()).unwrap().unwrap().v, held, "{label}");
|
||||||
|
}
|
||||||
|
("double", Cell::Double(_) | Cell::Int(_) | Cell::Uint(_) | Cell::Nil) => continue,
|
||||||
|
(kind, value) => panic!("{label}: unexpected vector row {kind} {value:?}"),
|
||||||
|
}
|
||||||
|
checked += 1;
|
||||||
|
}
|
||||||
|
assert!(checked > 10, "the vectors were read, not skipped: {checked}");
|
||||||
|
}
|
||||||
|
|
||||||
|
type OutcomeRow = (String, Option<String>, String, Option<String>, Option<String>);
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vectors_stores_hold_their_verdicts() {
|
||||||
|
let bytes = std::fs::read(vectors().join("outcomes.msgpack")).unwrap();
|
||||||
|
let rows: Vec<OutcomeRow> = rmp_serde::from_slice(&bytes).unwrap();
|
||||||
|
assert!(rows.len() >= 3);
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
for (file, alias, verdict, k1, k2) in rows {
|
||||||
|
let path = dir.path().join(&file);
|
||||||
|
std::fs::copy(vectors().join(&file), &path).unwrap();
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
let attach = |cache: &mut CultCache| {
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
};
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
let opened = match alias.as_deref() {
|
||||||
|
Some("Sku") => {
|
||||||
|
cache.register_entry_type::<Unique>().unwrap();
|
||||||
|
attach(&mut cache)
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
cache.register_entry_type::<Twin>().unwrap();
|
||||||
|
attach(&mut cache)
|
||||||
|
}
|
||||||
|
Some(other) => panic!("unexpected vector alias {other}"),
|
||||||
|
};
|
||||||
|
match verdict.as_str() {
|
||||||
|
"open" => opened.unwrap_or_else(|error| panic!("{file} / {alias:?} should open: {error:#}")),
|
||||||
|
"refused" => assert_eq!(
|
||||||
|
error_text(opened),
|
||||||
|
refusal(&k1.unwrap(), &k2.unwrap(), "tests.unique_item", "Sku"),
|
||||||
|
"{file}"
|
||||||
|
),
|
||||||
|
other => panic!("unexpected verdict {other}"),
|
||||||
|
}
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), before, "{file} is byte-identical");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One op of a refusals row: `["open", file-or-nil]`, `["pull", file]` or `["commit", [[key, schema, sku, second], ...]]`.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum Arg {
|
||||||
|
Records(Vec<(String, String, String, String)>),
|
||||||
|
File(Option<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
type RefusalRow = (String, Vec<(String, Arg)>, Option<(String, String, String, String)>);
|
||||||
|
|
||||||
|
/// Rows `[label, ops, refusal]`: the pair a refusal names is a function of the state the admission would leave
|
||||||
|
/// (`tests/vectors/unique-index/README.md`, RULE), whatever the staging, file or pull order, key case or alias case.
|
||||||
|
#[test]
|
||||||
|
fn vectors_refusals_hold_the_rule() {
|
||||||
|
let bytes = std::fs::read(vectors().join("refusals.msgpack")).unwrap();
|
||||||
|
let rows: Vec<RefusalRow> = rmp_serde::from_slice(&bytes).unwrap();
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let mut walked = 0;
|
||||||
|
for (index, (label, ops, refused)) in rows.into_iter().enumerate() {
|
||||||
|
let path = dir.path().join(format!("row-{index}.cc"));
|
||||||
|
let mut cache: Option<CultCache> = None;
|
||||||
|
let mut failure = None;
|
||||||
|
for (at, (kind, arg)) in ops.iter().enumerate() {
|
||||||
|
let last = at + 1 == ops.len();
|
||||||
|
let mut before = path.exists().then(|| std::fs::read(&path).unwrap());
|
||||||
|
let result = match (kind.as_str(), arg) {
|
||||||
|
("open", Arg::File(file)) => {
|
||||||
|
if let Some(file) = file {
|
||||||
|
std::fs::copy(vectors().join(file), &path).unwrap();
|
||||||
|
}
|
||||||
|
let mut opened = CultCache::new();
|
||||||
|
opened.register_entry_type::<Unique>().unwrap();
|
||||||
|
opened.register_entry_type::<UqPair>().unwrap();
|
||||||
|
opened.register_entry_type::<UqAlias>().unwrap();
|
||||||
|
opened.register_entry_type::<UqWide>().unwrap();
|
||||||
|
opened.register_entry_type::<UqAlpha>().unwrap();
|
||||||
|
opened.register_entry_type::<UqBeta>().unwrap();
|
||||||
|
opened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
let pulled = opened.pull_all_backing_stores();
|
||||||
|
cache = Some(opened);
|
||||||
|
pulled
|
||||||
|
}
|
||||||
|
("pull", Arg::File(Some(file))) => {
|
||||||
|
std::fs::copy(vectors().join(file), &path).unwrap();
|
||||||
|
before = Some(std::fs::read(&path).unwrap());
|
||||||
|
cache.as_mut().unwrap().pull_all_backing_stores()
|
||||||
|
}
|
||||||
|
("commit", Arg::Records(records)) => {
|
||||||
|
let cache = cache.as_mut().unwrap();
|
||||||
|
let staged: Vec<_> = records
|
||||||
|
.iter()
|
||||||
|
.map(|(key, schema, sku, second)| {
|
||||||
|
let (sku, second) = (sku.clone(), second.clone());
|
||||||
|
match schema.as_str() {
|
||||||
|
"tests.unique_item" => cache.prepare_entry(key, &Unique { sku, category: second }).map(|(envelope, _)| envelope),
|
||||||
|
"tests.uq_pair" => cache.prepare_entry(key, &UqPair { sku, alt: second }).map(|(envelope, _)| envelope),
|
||||||
|
"tests.uq_alias" => cache.prepare_entry(key, &UqAlias { sku, alt: second }).map(|(envelope, _)| envelope),
|
||||||
|
"tests.uq_wide" => cache.prepare_entry(key, &UqWide { sku, alt: second }).map(|(envelope, _)| envelope),
|
||||||
|
"tests.uq_alpha" => cache.prepare_entry(key, &UqAlpha { sku, second }).map(|(envelope, _)| envelope),
|
||||||
|
"tests.uq_beta" => cache.prepare_entry(key, &UqBeta { sku, second }).map(|(envelope, _)| envelope),
|
||||||
|
other => panic!("{label}: unexpected schema {other}"),
|
||||||
|
}
|
||||||
|
.unwrap()
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
cache.put_prepared_batch(staged)
|
||||||
|
}
|
||||||
|
other => panic!("{label}: unexpected op {:?}", other.0),
|
||||||
|
};
|
||||||
|
if last {
|
||||||
|
failure = Some((result, before));
|
||||||
|
} else {
|
||||||
|
result.unwrap_or_else(|error| panic!("{label}: op {at} should succeed: {error:#}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (result, before) = failure.unwrap();
|
||||||
|
match refused {
|
||||||
|
None => result.unwrap_or_else(|error| panic!("{label}: should succeed: {error:#}")),
|
||||||
|
Some((k1, k2, schema, alias)) => {
|
||||||
|
let error = result.err().unwrap_or_else(|| panic!("{label}: should refuse"));
|
||||||
|
assert_eq!(format!("{error:#}"), refusal(&k1, &k2, &schema, &alias), "{label}");
|
||||||
|
if let Some(before) = before {
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), before, "{label}: the file is byte-identical");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walked += 1;
|
||||||
|
}
|
||||||
|
assert_eq!(walked, 37);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A pull replaces the holder table with the stored image's: a value another writer changed is released, not kept.
|
||||||
|
#[test]
|
||||||
|
fn a_second_pull_replaces_the_holders_with_the_stored_image() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("pull.cc");
|
||||||
|
let mut cache = cache_at(&path);
|
||||||
|
cache.put("a", &unique("S")).unwrap();
|
||||||
|
|
||||||
|
cache_at(&path).put("a", &unique("T")).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), None, "the replaced value is no longer held");
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "T").unwrap(), cache.get::<Unique>("a").unwrap());
|
||||||
|
cache.put("b", &unique("S")).unwrap();
|
||||||
|
assert!(cache.put("c", &unique("T")).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
// The projector decides what held records hold: declaring an index for a type that already has records derives
|
||||||
|
// their holders, or refuses the registration when they collide.
|
||||||
|
#[test]
|
||||||
|
fn registering_an_index_over_held_records_derives_their_holders() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Twin>().unwrap();
|
||||||
|
cache.put("a", &Twin { sku: "S".into(), category: "x".into() }).unwrap();
|
||||||
|
cache.put("b", &Twin { sku: "T".into(), category: "x".into() }).unwrap();
|
||||||
|
|
||||||
|
cache.register_entry_type::<Unique>().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), cache.get::<Unique>("a").unwrap());
|
||||||
|
assert!(cache.get_by_index::<Unique>("Sku", "S").unwrap().is_some());
|
||||||
|
assert!(cache.put("c", &unique("S")).is_err(), "a held value is refused");
|
||||||
|
cache.put("c", &unique("U")).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn registering_an_index_over_colliding_held_records_is_refused_and_changes_nothing() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Twin>().unwrap();
|
||||||
|
cache.put("a", &Twin { sku: "S".into(), category: "x".into() }).unwrap();
|
||||||
|
cache.put("b", &Twin { sku: "S".into(), category: "y".into() }).unwrap();
|
||||||
|
|
||||||
|
let text = error_text(cache.register_entry_type::<Unique>());
|
||||||
|
assert_eq!(text, refusal("a", "b", "tests.unique_item", "Sku"));
|
||||||
|
|
||||||
|
// Still the unindexed declaration: duplicates keep landing, nothing is looked up.
|
||||||
|
cache.put("c", &Twin { sku: "S".into(), category: "z".into() }).unwrap();
|
||||||
|
assert_eq!(cache.get_all_with_keys::<Twin>().unwrap().len(), 3);
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same type and schema declaring Sku as an integer: the held text cannot be decoded as one.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "unique_item", schema = "tests.unique_item")]
|
||||||
|
struct Retyped {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A refused re-registration leaves the previous declaration in force: it keeps judging, so a later duplicate is refused.
|
||||||
|
#[test]
|
||||||
|
fn a_refused_reregistration_keeps_the_previous_index_judging() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Unique>().unwrap();
|
||||||
|
cache.put("a", &unique("S")).unwrap();
|
||||||
|
cache.put("b", &unique("T")).unwrap();
|
||||||
|
|
||||||
|
// A different declaration over the same type (Category is the unique alias) collides on held records.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "unique_item", schema = "tests.unique_item")]
|
||||||
|
struct ByCat {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
sku: String,
|
||||||
|
#[cultcache(key = 1, index = "Cat")]
|
||||||
|
category: String,
|
||||||
|
}
|
||||||
|
assert!(cache.register_entry_type::<ByCat>().is_err(), "a and b both hold category c");
|
||||||
|
|
||||||
|
assert!(cache.put("z", &unique("S")).is_err(), "the previous index still judges");
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), cache.get::<Unique>("a").unwrap());
|
||||||
|
cache.put("z", &unique("U")).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// A held record the new declaration cannot decode refuses the registration; the unindexed declaration stays.
|
||||||
|
#[test]
|
||||||
|
fn registering_an_index_over_an_undecodable_held_record_is_refused_and_changes_nothing() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Twin>().unwrap();
|
||||||
|
cache.put("a", &Twin { sku: "S".into(), category: "x".into() }).unwrap();
|
||||||
|
|
||||||
|
assert!(cache.register_entry_type::<Retyped>().is_err());
|
||||||
|
|
||||||
|
cache.put("b", &Twin { sku: "S".into(), category: "y".into() }).unwrap();
|
||||||
|
assert_eq!(cache.get_all_with_keys::<Twin>().unwrap().len(), 2, "no index judges the duplicate");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dropping the declaration releases the holders.
|
||||||
|
#[test]
|
||||||
|
fn registering_without_the_index_releases_the_holders() {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Unique>().unwrap();
|
||||||
|
cache.put("a", &unique("S")).unwrap();
|
||||||
|
cache.register_entry_type::<Twin>().unwrap();
|
||||||
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap(), None, "no declaration, no holders");
|
||||||
|
cache.put("b", &Twin { sku: "S".into(), category: "x".into() }).unwrap();
|
||||||
|
assert!(cache.register_entry_type::<Unique>().is_err(), "the duplicate now written is a held violation");
|
||||||
|
}
|
||||||
|
|
@ -7,12 +7,23 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
|
- A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
||||||
- `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank
|
- `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank
|
||||||
value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical
|
value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical
|
||||||
text (booleans `True`/`False`, integers as digits, doubles as the exact expansion, blank values never indexed), so
|
text (booleans `True`/`False`, integers as digits, doubles as the exact expansion, blank values never indexed), so
|
||||||
an empty string is no longer a value. `registerIndex` over held duplicates installs nothing. A new `roles` map names
|
an empty string is no longer a value. `registerIndex` over held duplicates installs nothing. A new `roles` map names
|
||||||
the many-row selection aliases (never indexed for lookup, same catalog `indexAlias`). The per-key migration rule is
|
the many-row selection aliases (never indexed for lookup, same catalog `indexAlias`). The per-key migration rule is
|
||||||
in the README; classify every key before the CultLib your tools load is updated. The refusal verdict is the contract, its text is informational.
|
in the README; classify every key before the CultLib your tools load is updated. The refusal verdict is the contract, its text is informational.
|
||||||
|
- The single-file reader (v1 and the legacy envelope array) refuses a store that holds two records under one key as `StoreUnreadableError`; a store that opened before may now be refused. Repair it by removing the extra record with the runtime that wrote it.
|
||||||
|
- A `put`, `push` or `pushAll` that would land a second type under a held key (against the file, or within one batch) is refused with `SchemaConflictError`, and nothing is written. The written type may replace the record when its catalog entry carries the stored schema id as its own or lists it compatible (a type renamed under a stable id restamps the record, as C# does); otherwise retyping a key is a delete, then a put. The C# allowance for an undeclared older id of the same schema name that reads whole is not ported: a same-named type keeps passing, including one that drops persisted slots, which C# refuses.
|
||||||
|
- Same-type writes replace by key; in one `pushAll` batch the last entry for a key wins.
|
||||||
|
- A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`.
|
||||||
|
- The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string.
|
||||||
|
- A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
|
@ -27,7 +38,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
- A single-file store written as `cultcache.store.v3` is read and written: a record that holds an element id (an owned list element with an `[CultElementId]` member, carried through every write) marks the store v3, and a v3 store is carried through a rewrite with its ids. A runtime from before this change refuses a v3 store by name.
|
- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name.
|
||||||
- A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v<digits>`.
|
- A single-file store refuses a zero-byte file as `StoreUnreadableError`; only a path with nothing at it is an empty store. A header this runtime does not read is refused, and the refusal echoes the header only as `cultcache.store.v<digits>`.
|
||||||
- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. A full `pushAll` over a file with no readable stamp does not replace it: the reader refuses it. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole.
|
- A write over a file the reader refuses (`StoreUnreadableError`, from `push`, `pushAll` and `delete`) refuses and leaves the file as it is. A flush used to replace such a file. A write set cannot merge with a disk it cannot read, so the file is repaired or removed by its owner before the store writes again. A full `pushAll` over a file with no readable stamp does not replace it: the reader refuses it. The stamp rule has one owner, `refuseUnreadFormat`, asked only by the one reader that every write consults under the lock. The reader judges the stamp from the header before it decodes the body, so a newer store is refused by name whether or not its body is whole.
|
||||||
- A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store.
|
- A dangling symbolic link at the store path is an I/O error with code `ENOENT`, not an empty store.
|
||||||
|
|
|
||||||
|
|
@ -63,6 +63,10 @@ const payload = encodeDocumentPayload(noteDocument, note); // DocumentPayload
|
||||||
|
|
||||||
Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. The whole cache without the single-file store is `@gamecult/cultcache-ts/browser`, which is also what bundlers resolve for the package root (`browser` condition); bring your own `CacheBackingStore`.
|
Browsers import the same names from `@gamecult/cultcache-ts/document`, which holds no Node built-ins. The whole cache without the single-file store is `@gamecult/cultcache-ts/browser`, which is also what bundlers resolve for the package root (`browser` condition); bring your own `CacheBackingStore`.
|
||||||
|
|
||||||
|
## Store format and rollback
|
||||||
|
|
||||||
|
A store whose elements hold an id is marked `cultcache.store.v3` by the C# runtime. This runtime reads a v3 store, keeps the header it read and carries the ids through a rewrite; it never drops the mark, so it has no carry-back. A cultcache-ts from before element ids refuses a v3 store by name, so rolling back to it after a store has been written as v3 is one-way: restore from a copy taken before upgrading, and upgrade every reader before any writer. (The C# line is different: CultLib 1.x does not refuse; see the GameCult.Caching README.)
|
||||||
|
|
||||||
## `.cc` Inspection
|
## `.cc` Inspection
|
||||||
|
|
||||||
CultCache files may use the `.cc` extension. The bytes are still the canonical
|
CultCache files may use the `.cc` extension. The bytes are still the canonical
|
||||||
|
|
@ -227,6 +231,16 @@ its exact decimal expansion; NaN and infinities have no value. `roles` name sele
|
||||||
member's catalog `indexAlias`, so the catalog bytes are the same either way); they imply nothing about uniqueness and
|
member's catalog `indexAlias`, so the catalog bytes are the same either way); they imply nothing about uniqueness and
|
||||||
are never indexed for lookup, so `getByIndex` on a role answers `undefined`. An alias is declared in one of the two.
|
are never indexed for lookup, so `getByIndex` on a role answers `undefined`. An alias is declared in one of the two.
|
||||||
|
|
||||||
|
### One record per key, and foreign records
|
||||||
|
|
||||||
|
A cache holds one record per key whatever its type (variants:ruling:cache-refuses-retype). A write of a type that does not
|
||||||
|
own the held record's schema id (it carries that id as its own or lists it compatible) is refused with
|
||||||
|
`SchemaConflictError` naming the key, with or without a store; retyping a key is `delete` then `put`. A store that holds a
|
||||||
|
record no registered type claims still loads: the cache does not serve it, `foreignRecords()` lists it (`key`,
|
||||||
|
`schemaId`, `schemaName`, `storedAt`, by key ordinal), every write keeps its bytes, and a `put` or `delete` at its key is
|
||||||
|
refused. Registering a type that claims its schema id serves it at the next `pullAllBackingStores`. Two stores that hold
|
||||||
|
one key refuse the load and admit nothing.
|
||||||
|
|
||||||
### Migrating an existing `indexes` key
|
### Migrating an existing `indexes` key
|
||||||
|
|
||||||
Apply per key, and do it BEFORE the CultLib checkout your tools load is rebuilt or updated. A consumer that loads
|
Apply per key, and do it BEFORE the CultLib checkout your tools load is rebuilt or updated. A consumer that loads
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
// (node:fs, koffi). Bring your own CacheBackingStore, or none.
|
// (node:fs, koffi). Bring your own CacheBackingStore, or none.
|
||||||
export * as codecs from "./codec";
|
export * as codecs from "./codec";
|
||||||
export type { Codec, SlotsCodec } from "./codec";
|
export type { Codec, SlotsCodec } from "./codec";
|
||||||
export { CultCache, CultCacheBuilder } from "./cult-cache";
|
export { CultCache, CultCacheBuilder, type CultCacheForeignRecord } from "./cult-cache";
|
||||||
export { inspectCultCacheBytes } from "./cult-cache-inspector";
|
export { inspectCultCacheBytes } from "./cult-cache-inspector";
|
||||||
export {
|
export {
|
||||||
decodeDocumentPayload,
|
decodeDocumentPayload,
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import {
|
import {
|
||||||
|
declaresRole,
|
||||||
decodeDocumentPayload,
|
decodeDocumentPayload,
|
||||||
documentPayloadBytes,
|
documentPayloadBytes,
|
||||||
documentPayloadFromBytes,
|
documentPayloadFromBytes,
|
||||||
|
|
@ -8,7 +9,7 @@ import {
|
||||||
} from "./document";
|
} from "./document";
|
||||||
import { resolveDocumentFormatter } from "./document-formatter";
|
import { resolveDocumentFormatter } from "./document-formatter";
|
||||||
import type { DocumentPayload } from "./document";
|
import type { DocumentPayload } from "./document";
|
||||||
import { SchemaConflictError } from "./store-format";
|
import { SchemaConflictError, schemaIdForEnvelope, writtenOwns } from "./store-format";
|
||||||
import type {
|
import type {
|
||||||
AnyCultCacheDocumentDefinition,
|
AnyCultCacheDocumentDefinition,
|
||||||
CacheBackingStore,
|
CacheBackingStore,
|
||||||
|
|
@ -46,6 +47,7 @@ type EntryLookups = {
|
||||||
};
|
};
|
||||||
|
|
||||||
type HydratedEntry = CultCacheEnvelope & {
|
type HydratedEntry = CultCacheEnvelope & {
|
||||||
|
catalogEntry: NonNullable<CultCacheEnvelope["catalogEntry"]>;
|
||||||
value: unknown;
|
value: unknown;
|
||||||
lookups: EntryLookups;
|
lookups: EntryLookups;
|
||||||
};
|
};
|
||||||
|
|
@ -62,6 +64,17 @@ type HydratedState = {
|
||||||
// delete of that global removes it from the home store. Remove once no store holds a global under
|
// delete of that global removes it from the home store. Remove once no store holds a global under
|
||||||
// a key other than GLOBAL_KEY.
|
// a key other than GLOBAL_KEY.
|
||||||
readonly legacyGlobals: Map<string, CultCacheEnvelope>;
|
readonly legacyGlobals: Map<string, CultCacheEnvelope>;
|
||||||
|
// The records the last pull found that no registered type claims. Derived from the pull and replaced with the state; it
|
||||||
|
// decides nothing but the refusal of a write or a removal at its keys.
|
||||||
|
readonly foreign: CultCacheForeignRecord[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/** A record a store carries that no registered type claims: the cache does not serve it, keeps its bytes, and refuses a write or a removal at its key. */
|
||||||
|
export type CultCacheForeignRecord = {
|
||||||
|
readonly key: string;
|
||||||
|
readonly schemaId: string;
|
||||||
|
readonly schemaName: string;
|
||||||
|
readonly storedAt: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Lookups for every held record of one type under candidate accessors, derived before anything is installed.
|
// Lookups for every held record of one type under candidate accessors, derived before anything is installed.
|
||||||
|
|
@ -79,6 +92,7 @@ function emptyHydratedState(): HydratedState {
|
||||||
indexToKeyMaps: new Map(),
|
indexToKeyMaps: new Map(),
|
||||||
globalKeys: new Map(),
|
globalKeys: new Map(),
|
||||||
legacyGlobals: new Map(),
|
legacyGlobals: new Map(),
|
||||||
|
foreign: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -240,6 +254,10 @@ export class CultCache {
|
||||||
throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`);
|
throw new Error(`CultCache index names for type "${definition.type}" must be non-empty.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (declaresRole(definition, indexName)) {
|
||||||
|
throw new Error(`Index "${indexName}" of ${definition.type} is declared as a role; a role is never a unique index.`);
|
||||||
|
}
|
||||||
|
|
||||||
const registered = this.#requireDefinition(definition);
|
const registered = this.#requireDefinition(definition);
|
||||||
const indexAccessors = new Map(registered.indexAccessors);
|
const indexAccessors = new Map(registered.indexAccessors);
|
||||||
indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor));
|
indexAccessors.set(indexName, this.#compileAccessor(definition.type, `index "${indexName}"`, accessor));
|
||||||
|
|
@ -274,11 +292,14 @@ export class CultCache {
|
||||||
for (const entry of entries) {
|
for (const entry of entries) {
|
||||||
const registered = this.#resolveDefinitionForEnvelope(entry);
|
const registered = this.#resolveDefinitionForEnvelope(entry);
|
||||||
if (!registered) {
|
if (!registered) {
|
||||||
throw new Error(
|
// A store may hold what this cache has no type for; it is carried, not served, and not a reason to refuse the load.
|
||||||
entry.schemaId
|
next.foreign.push({
|
||||||
? `No schema is registered for persisted schema id "${entry.schemaId}".`
|
key: entry.key,
|
||||||
: `No schema is registered for persisted document type "${entry.type}".`,
|
schemaId: schemaIdForEnvelope(entry),
|
||||||
);
|
schemaName: entry.catalogEntry?.schemaName ?? entry.type,
|
||||||
|
storedAt: entry.storedAt,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const type = registered.definition.type;
|
const type = registered.definition.type;
|
||||||
|
|
@ -307,10 +328,37 @@ export class CultCache {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const hydrated = this.#admitEntry(next, registered, admitted, value);
|
const hydrated = this.#admitEntry(next, registered, admitted, value);
|
||||||
|
const other = this.#heldUnderOtherType(next, type, hydrated.key);
|
||||||
|
if (other !== undefined) {
|
||||||
|
throw new Error(`Record ${hydrated.key} is held by two stores, as type "${other.type}" and as type "${type}".`);
|
||||||
|
}
|
||||||
this.#installEntry(next, registered, hydrated);
|
this.#installEntry(next, registered, hydrated);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const foreignByKey = new Map<string, string>();
|
||||||
|
for (const record of next.foreign) {
|
||||||
|
const first = foreignByKey.get(record.key);
|
||||||
|
if (first !== undefined) {
|
||||||
|
throw new Error(`Record ${record.key} is held by two stores, as schema "${first}" and as schema "${record.schemaName}".`);
|
||||||
|
}
|
||||||
|
foreignByKey.set(record.key, record.schemaName);
|
||||||
|
const other = [...this.#definitions.keys()].map((type) => next.entries.get(this.#entryId(type, record.key))).find((held) => held !== undefined);
|
||||||
|
if (other !== undefined) {
|
||||||
|
throw new Error(`Record ${record.key} is held by two stores, as type "${other.type}" and as schema "${record.schemaName}".`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Judged whole, once: the refusal names the least violation of the state this load would leave, whatever the
|
||||||
|
// order of the stores and of the entries in them.
|
||||||
|
refuseLeastIndexSharing(function* () {
|
||||||
|
for (const entry of next.entries.values()) {
|
||||||
|
for (const [alias, value] of entry.lookups.indexes) {
|
||||||
|
yield { type: entry.type, schemaName: entry.catalogEntry.schemaName, alias, value, key: entry.key };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}());
|
||||||
|
|
||||||
this.#hydrated = next;
|
this.#hydrated = next;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -537,6 +585,11 @@ export class CultCache {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What the last pull found that no registered type claims, by key ordinal. A later registration claims a record at the next pull.
|
||||||
|
foreignRecords(): readonly CultCacheForeignRecord[] {
|
||||||
|
return [...this.#hydrated.foreign].sort((left, right) => compareOrdinal(left.key, right.key));
|
||||||
|
}
|
||||||
|
|
||||||
snapshot(): CultCacheEnvelope[] {
|
snapshot(): CultCacheEnvelope[] {
|
||||||
return [...this.#hydrated.entries.values()].map((entry) => this.#toEnvelope(entry));
|
return [...this.#hydrated.entries.values()].map((entry) => this.#toEnvelope(entry));
|
||||||
}
|
}
|
||||||
|
|
@ -646,16 +699,73 @@ export class CultCache {
|
||||||
// is touched; once the store accepts, applying to the cache has no fallible step.
|
// is touched; once the store accepts, applying to the cache has no fallible step.
|
||||||
async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise<void> {
|
async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise<void> {
|
||||||
const home = this.#homeStore(entry.type);
|
const home = this.#homeStore(entry.type);
|
||||||
|
const retyped = this.#refuseRetype(entry);
|
||||||
|
if (retyped !== undefined && this.#homeStore(retyped.type) !== home) {
|
||||||
|
// The write owns the held record's id and would replace it, but the held record is in another store; the push
|
||||||
|
// reaches only this type's home, so the old record would stay in its own file and load as a second holder.
|
||||||
|
throw new SchemaConflictError(
|
||||||
|
schemaIdForEnvelope(entry),
|
||||||
|
[retyped.type, entry.type],
|
||||||
|
entry.key,
|
||||||
|
`CultCache holds record "${entry.key}" as type "${retyped.type}" in another store than the home of type "${entry.type}", so the rename cannot replace it; remove the record first.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value);
|
const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value);
|
||||||
|
this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups);
|
||||||
const legacy = this.#hydrated.legacyGlobals.get(entry.type);
|
const legacy = this.#hydrated.legacyGlobals.get(entry.type);
|
||||||
await home?.push(entry);
|
await home?.push(entry);
|
||||||
if (home && legacy) {
|
if (home && legacy) {
|
||||||
await this.#retireLegacyGlobal(home, legacy, entry);
|
await this.#retireLegacyGlobal(home, legacy, entry);
|
||||||
}
|
}
|
||||||
this.#hydrated.legacyGlobals.delete(entry.type);
|
this.#hydrated.legacyGlobals.delete(entry.type);
|
||||||
|
if (retyped !== undefined) {
|
||||||
|
// The record the write restamped (it owns the held id) is the same record: one key, one held type.
|
||||||
|
this.#removeEntry(this.#hydrated, this.#definitions.get(retyped.type)!, retyped);
|
||||||
|
}
|
||||||
this.#installEntry(this.#hydrated, registered, hydrated);
|
this.#installEntry(this.#hydrated, registered, hydrated);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The one key-identity judge of the cache: may this write land at its key, given what the cache holds there under
|
||||||
|
// every registered type and the keys its stores carry foreign. The store judges the file with the same predicate
|
||||||
|
// (writtenOwns); this judges the cache, which a store-less cache has no other judge for. Returns the held record of
|
||||||
|
// another type the write owns and so replaces, if any. A refusal names the key and schemas, never a payload.
|
||||||
|
#refuseRetype(entry: CultCacheEnvelope): HydratedEntry | undefined {
|
||||||
|
this.#refuseForeignKey(entry.key, entry.type, "written");
|
||||||
|
const held = this.#heldUnderOtherType(this.#hydrated, entry.type, entry.key);
|
||||||
|
if (held === undefined || writtenOwns(entry, schemaIdForEnvelope(held))) {
|
||||||
|
return held;
|
||||||
|
}
|
||||||
|
throw new SchemaConflictError(
|
||||||
|
schemaIdForEnvelope(entry),
|
||||||
|
[held.type, entry.type],
|
||||||
|
entry.key,
|
||||||
|
`CultCache holds record "${entry.key}" as type "${held.type}", so type "${entry.type}" cannot be written under the same key; remove the record first.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#refuseForeignKey(key: string, type: string, what: "written" | "removed"): void {
|
||||||
|
const foreign = this.#hydrated.foreign.find((record) => record.key === key);
|
||||||
|
if (foreign !== undefined) {
|
||||||
|
throw new SchemaConflictError(
|
||||||
|
foreign.schemaId,
|
||||||
|
[foreign.schemaName],
|
||||||
|
key,
|
||||||
|
`CultCache store carries record "${key}" as schema "${foreign.schemaName}" (${foreign.schemaId}), which no registered type claims, so type "${type}" cannot be ${what} at that key.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The record another registered type holds at the key, if any.
|
||||||
|
#heldUnderOtherType(state: HydratedState, type: string, key: string): HydratedEntry | undefined {
|
||||||
|
for (const other of this.#definitions.keys()) {
|
||||||
|
const held = other === type ? undefined : state.entries.get(this.#entryId(other, key));
|
||||||
|
if (held !== undefined) {
|
||||||
|
return held;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
// The first write of a global loaded from a legacy key removes that record in the same write. If the
|
// The first write of a global loaded from a legacy key removes that record in the same write. If the
|
||||||
// removal fails, the new record is taken back out, so the store keeps only the legacy record.
|
// removal fails, the new record is taken back out, so the store keeps only the legacy record.
|
||||||
async #retireLegacyGlobal(home: CacheBackingStore, legacy: CultCacheEnvelope, written: CultCacheEnvelope): Promise<void> {
|
async #retireLegacyGlobal(home: CacheBackingStore, legacy: CultCacheEnvelope, written: CultCacheEnvelope): Promise<void> {
|
||||||
|
|
@ -674,6 +784,9 @@ export class CultCache {
|
||||||
const registered = this.#requireDefinition(definition);
|
const registered = this.#requireDefinition(definition);
|
||||||
const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key));
|
const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key));
|
||||||
if (!entry) {
|
if (!entry) {
|
||||||
|
// variants:ruling:unheld-removal-refused-at-commit: a key a store carries foreign is refused, a key held under
|
||||||
|
// another type or by nobody is the existing false.
|
||||||
|
this.#refuseForeignKey(key, definition.type, "removed");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -898,7 +1011,6 @@ export class CultCache {
|
||||||
}
|
}
|
||||||
|
|
||||||
const lookups = this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value);
|
const lookups = this.#lookupsFor(registered.nameAccessor, registered.indexAccessors, value);
|
||||||
this.#refuseIndexSharing(state, entry.type, registered.catalogEntry.schemaName, entry.key, lookups);
|
|
||||||
return {
|
return {
|
||||||
...entry,
|
...entry,
|
||||||
payload: this.#cloneBytes(entry.payload),
|
payload: this.#cloneBytes(entry.payload),
|
||||||
|
|
@ -990,7 +1102,6 @@ export class CultCache {
|
||||||
const entryLookups: Array<[HydratedEntry, EntryLookups]> = [];
|
const entryLookups: Array<[HydratedEntry, EntryLookups]> = [];
|
||||||
const nameMap = nameAccessor ? new Map<string, string>() : undefined;
|
const nameMap = nameAccessor ? new Map<string, string>() : undefined;
|
||||||
const indexMaps = new Map<string, Map<string, string>>();
|
const indexMaps = new Map<string, Map<string, string>>();
|
||||||
const sharing = new Map<string, string[]>();
|
|
||||||
for (const indexName of indexAccessors.keys()) {
|
for (const indexName of indexAccessors.keys()) {
|
||||||
indexMaps.set(indexName, new Map());
|
indexMaps.set(indexName, new Map());
|
||||||
}
|
}
|
||||||
|
|
@ -1003,21 +1114,16 @@ export class CultCache {
|
||||||
}
|
}
|
||||||
for (const [indexName, indexValue] of lookups.indexes) {
|
for (const [indexName, indexValue] of lookups.indexes) {
|
||||||
const map = indexMaps.get(indexName)!;
|
const map = indexMaps.get(indexName)!;
|
||||||
const holder = map.get(indexValue);
|
if (!map.has(indexValue)) map.set(indexValue, entry.key);
|
||||||
if (holder === undefined) {
|
|
||||||
map.set(indexValue, entry.key);
|
|
||||||
} else {
|
|
||||||
sharing.set(`${indexName}\u0000${indexValue}`, [...(sharing.get(`${indexName}\u0000${indexValue}`) ?? [holder]), entry.key]);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Judged whole: any value held by two records refuses, naming the two smallest of its keys. Held records already
|
// Judged whole, by the same judge as a load: the least violation of the state the registration would leave.
|
||||||
// passed the judge for every alias installed before, so only the alias being registered can be shared here.
|
refuseLeastIndexSharing(function* () {
|
||||||
for (const [id, keys] of sharing) {
|
for (const [entry, lookups] of entryLookups) {
|
||||||
keys.sort(compareOrdinal);
|
for (const [alias, value] of lookups.indexes) yield { type, schemaName, alias, value, key: entry.key };
|
||||||
throw indexSharingRefusal(keys[0], keys[1], schemaName, id.slice(0, id.indexOf("\u0000")));
|
}
|
||||||
}
|
}());
|
||||||
|
|
||||||
return { entryLookups, nameMap, indexMaps };
|
return { entryLookups, nameMap, indexMaps };
|
||||||
}
|
}
|
||||||
|
|
@ -1094,6 +1200,34 @@ function compareOrdinal(a: string, b: string): number {
|
||||||
return a < b ? -1 : a > b ? 1 : 0;
|
return a < b ? -1 : a > b ? 1 : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The whole-state judge (variants:question:unique-index-refusal-pair-rule): over every (type, alias, value) held at two
|
||||||
|
// or more distinct keys, throws for the one least by (alias, least key, second-least key, schema name), each compared by
|
||||||
|
// UTF-16 code unit. A function of the state, never of the order the rows arrive in.
|
||||||
|
function refuseLeastIndexSharing(rows: Iterable<{ type: string; schemaName: string; alias: string; value: string; key: string }>): void {
|
||||||
|
const groups = new Map<string, { schemaName: string; alias: string; keys: string[] }>();
|
||||||
|
for (const { type, schemaName, alias, value, key } of rows) {
|
||||||
|
const id = JSON.stringify([type, alias, value]);
|
||||||
|
const group = groups.get(id);
|
||||||
|
if (group) group.keys.push(key);
|
||||||
|
else groups.set(id, { schemaName, alias, keys: [key] });
|
||||||
|
}
|
||||||
|
let least: { schemaName: string; alias: string; first: string; second: string } | undefined;
|
||||||
|
for (const { schemaName, alias, keys } of groups.values()) {
|
||||||
|
if (keys.length < 2) continue;
|
||||||
|
keys.sort(compareOrdinal);
|
||||||
|
const candidate = { schemaName, alias, first: keys[0], second: keys[1] };
|
||||||
|
if (!least || compareLeast(candidate, least) < 0) least = candidate;
|
||||||
|
}
|
||||||
|
if (least) throw indexSharingRefusal(least.first, least.second, least.schemaName, least.alias);
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareLeast(
|
||||||
|
a: { schemaName: string; alias: string; first: string; second: string },
|
||||||
|
b: { schemaName: string; alias: string; first: string; second: string },
|
||||||
|
): number {
|
||||||
|
return compareOrdinal(a.alias, b.alias) || compareOrdinal(a.first, b.first) || compareOrdinal(a.second, b.second) || compareOrdinal(a.schemaName, b.schemaName);
|
||||||
|
}
|
||||||
|
|
||||||
function indexSharingRefusal(keyA: string, keyB: string, schemaName: string, alias: string): Error {
|
function indexSharingRefusal(keyA: string, keyB: string, schemaName: string, alias: string): Error {
|
||||||
const [first, second] = compareOrdinal(keyA, keyB) <= 0 ? [keyA, keyB] : [keyB, keyA];
|
const [first, second] = compareOrdinal(keyA, keyB) <= 0 ? [keyA, keyB] : [keyB, keyA];
|
||||||
return new Error(`Records ${first} and ${second} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`);
|
return new Error(`Records ${first} and ${second} (${schemaName}) both hold unique index '${alias}'; an index value names one record.`);
|
||||||
|
|
|
||||||
|
|
@ -131,6 +131,12 @@ export function encodeDocumentPayload(definition: AnyCultCacheDocumentDefinition
|
||||||
return documentPayloadFromBytes(bytes);
|
return documentPayloadFromBytes(bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Whether the definition declares `name` as a role alias (either `roles` form).
|
||||||
|
export function declaresRole(definition: AnyCultCacheDocumentDefinition, name: string): boolean {
|
||||||
|
const roles = definition.roles;
|
||||||
|
return Array.isArray(roles) ? roles.some((entry) => entry.name === name) : roles ? Object.hasOwn(roles, name) : false;
|
||||||
|
}
|
||||||
|
|
||||||
// The schema members of a document: the ones it declares, else those its codec derives, else none.
|
// The schema members of a document: the ones it declares, else those its codec derives, else none.
|
||||||
export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinition): readonly CultCacheSchemaCatalogMember[] {
|
export function resolveDocumentMembers(definition: AnyCultCacheDocumentDefinition): readonly CultCacheSchemaCatalogMember[] {
|
||||||
if (definition.members || !definition.codec) {
|
if (definition.members || !definition.codec) {
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ import { z } from "zod";
|
||||||
|
|
||||||
import { documentPayloadBytes, documentPayloadFromBytes } from "./document";
|
import { documentPayloadBytes, documentPayloadFromBytes } from "./document";
|
||||||
import { commitStoreFile, withStoreWriteLock } from "./store-file";
|
import { commitStoreFile, withStoreWriteLock } from "./store-file";
|
||||||
import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format";
|
import { STORE_FORMAT_VERSION, SchemaConflictError, type StoreFormat, StoreUnreadableError, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots, schemaIdForEnvelope, writtenOwns } from "./store-format";
|
||||||
import type {
|
import type {
|
||||||
CacheBackingStore,
|
CacheBackingStore,
|
||||||
CultCacheEnvelope,
|
CultCacheEnvelope,
|
||||||
|
|
@ -60,9 +60,8 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
||||||
const disk = this.#readDisk();
|
const disk = this.#readDisk();
|
||||||
this.#format = disk.format;
|
this.#format = disk.format;
|
||||||
const existing = disk.envelopes;
|
const existing = disk.envelopes;
|
||||||
const filtered = existing.filter(
|
requireOneTypePerKey(this.filePath, [entry], existing);
|
||||||
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
|
const filtered = existing.filter((candidate) => candidate.key !== entry.key);
|
||||||
);
|
|
||||||
await this.#writeAll([entry], filtered);
|
await this.#writeAll([entry], filtered);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -72,8 +71,10 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
||||||
const disk = this.#readDisk();
|
const disk = this.#readDisk();
|
||||||
this.#format = disk.format;
|
this.#format = disk.format;
|
||||||
const existing = disk.envelopes;
|
const existing = disk.envelopes;
|
||||||
|
// The record the file stores at the key is the one removed when the removal's schema owns its stored id: the
|
||||||
|
// arrived record's type is its schema's name, which a type of another name does not carry.
|
||||||
const filtered = existing.filter(
|
const filtered = existing.filter(
|
||||||
(candidate) => !(candidate.type === entry.type && candidate.key === entry.key),
|
(candidate) => !(candidate.key === entry.key && (schemaIdForEnvelope(candidate) === schemaIdForEnvelope(entry) || writtenOwns(entry, schemaIdForEnvelope(candidate)))),
|
||||||
);
|
);
|
||||||
await this.#writeAll([], filtered);
|
await this.#writeAll([], filtered);
|
||||||
});
|
});
|
||||||
|
|
@ -103,7 +104,9 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
||||||
this.#format = STORE_FORMAT_VERSION;
|
this.#format = STORE_FORMAT_VERSION;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.#writeAll(entries, []);
|
requireOneTypePerKey(this.filePath, entries, []);
|
||||||
|
// A key is held once: of same-type entries sharing a key in one batch, the last is kept.
|
||||||
|
await this.#writeAll([...new Map(entries.map((entry) => [entry.key, entry])).values()], []);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -137,6 +140,28 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A store holds a key once, and the reader refuses a file that holds it twice, so a write that would put a second type under a
|
||||||
|
// key (against a record on disk, or within its own batch) is refused before anything is written. Retyping a record is a delete
|
||||||
|
// of the old type followed by a put of the new one.
|
||||||
|
function requireOneTypePerKey(path: string, supplied: CultCacheEnvelope[], held: CultCacheEnvelope[]): void {
|
||||||
|
const typeByKey = new Map<string, CultCacheEnvelope>();
|
||||||
|
for (const entry of held) {
|
||||||
|
typeByKey.set(entry.key, entry);
|
||||||
|
}
|
||||||
|
for (const entry of supplied) {
|
||||||
|
const prior = typeByKey.get(entry.key);
|
||||||
|
if (prior !== undefined && prior.type !== entry.type && !writtenOwns(entry, schemaIdForEnvelope(prior))) {
|
||||||
|
throw new SchemaConflictError(
|
||||||
|
schemaIdForEnvelope(entry),
|
||||||
|
[prior.type, entry.type],
|
||||||
|
entry.key,
|
||||||
|
`CultCache store ${path} holds record "${entry.key}" as type "${prior.type}", so type "${entry.type}" cannot be written under the same key; remove the record first.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
typeByKey.set(entry.key, entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The one reader of a CultCache single-file store, for this package's store and for any consumer that must read a
|
* The one reader of a CultCache single-file store, for this package's store and for any consumer that must read a
|
||||||
* store file (an authority bundle, say) rather than decode it itself. A path with nothing at it is an absent, empty,
|
* store file (an authority bundle, say) rather than decode it itself. A path with nothing at it is an absent, empty,
|
||||||
|
|
@ -204,8 +229,9 @@ type DecodedSnapshot = {
|
||||||
};
|
};
|
||||||
|
|
||||||
function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] {
|
function encodeSnapshot(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[], format: StoreFormat): unknown[] {
|
||||||
const replaced = new Set(supplied.map((entry) => `${entry.type}::${entry.key}`));
|
// A key is held once: a supplied record replaces the arrived record under its key whatever the type.
|
||||||
const kept = arrived.filter((entry) => !replaced.has(`${entry.type}::${entry.key}`));
|
const replaced = new Set(supplied.map((entry) => entry.key));
|
||||||
|
const kept = arrived.filter((entry) => !replaced.has(entry.key));
|
||||||
const entries = [...kept, ...supplied];
|
const entries = [...kept, ...supplied];
|
||||||
const catalog = catalogEntriesFor([
|
const catalog = catalogEntriesFor([
|
||||||
...kept.map((envelope) => ({ envelope, registered: false })),
|
...kept.map((envelope) => ({ envelope, registered: false })),
|
||||||
|
|
@ -351,10 +377,6 @@ function defaultEntryFor(entry: CultCacheEnvelope): CultCacheSchemaCatalogEntry
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function schemaIdForEnvelope(entry: CultCacheEnvelope): string {
|
|
||||||
return entry.schemaId ?? entry.type;
|
|
||||||
}
|
|
||||||
|
|
||||||
function encodeCatalogEntry(entry: CultCacheSchemaCatalogEntry): unknown[] {
|
function encodeCatalogEntry(entry: CultCacheSchemaCatalogEntry): unknown[] {
|
||||||
return [
|
return [
|
||||||
entry.schemaId,
|
entry.schemaId,
|
||||||
|
|
@ -416,6 +438,7 @@ function decodeStoreFile(data: Uint8Array): Omit<SingleFileStoreRead, "absent">
|
||||||
throw new Error("CultCache file is not a recognized CultCache MessagePack store.");
|
throw new Error("CultCache file is not a recognized CultCache MessagePack store.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
requireUniqueKeys(legacy);
|
||||||
const normalized = legacy.map((entry) => ({ ...entry, payload: normalizePayload(entry.payload) }));
|
const normalized = legacy.map((entry) => ({ ...entry, payload: normalizePayload(entry.payload) }));
|
||||||
return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[] };
|
return { format: STORE_FORMAT_VERSION, envelopes: envelopeArraySchema.parse(normalized).map((entry) => ({ ...entry, payload: documentPayloadFromBytes(entry.payload) })) as CultCacheEnvelope[] };
|
||||||
}
|
}
|
||||||
|
|
@ -449,9 +472,21 @@ function decodeSnapshot(decoded: unknown): DecodedSnapshot | undefined {
|
||||||
}
|
}
|
||||||
|
|
||||||
const records = recordsRaw.map(decodeRecord);
|
const records = recordsRaw.map(decodeRecord);
|
||||||
|
requireUniqueKeys(records);
|
||||||
return { format: decoded[0], catalogBySchemaId, records };
|
return { format: decoded[0], catalogBySchemaId, records };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A store holds a key once, whatever the type: a repeat is refused in a v1 snapshot and in a legacy envelope array alike.
|
||||||
|
function requireUniqueKeys(records: Array<{ key: string }>): void {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const { key } of records) {
|
||||||
|
if (seen.has(key)) {
|
||||||
|
throw new Error(`CultCache record "${key}" appears more than once.`);
|
||||||
|
}
|
||||||
|
seen.add(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry {
|
function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry {
|
||||||
if (!Array.isArray(value)) {
|
if (!Array.isArray(value)) {
|
||||||
throw new Error("CultCache schema catalog entries must be MessagePack arrays.");
|
throw new Error("CultCache schema catalog entries must be MessagePack arrays.");
|
||||||
|
|
@ -479,12 +514,16 @@ function decodeCatalogEntry(value: unknown): CultCacheSchemaCatalogEntry {
|
||||||
throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid members.`);
|
throw new Error(`CultCache schema catalog entry "${schemaId}" has invalid members.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (typeof canonicalSchemaJson !== "string") {
|
||||||
|
throw new Error(`CultCache schema catalog entry "${schemaId}" has an invalid canonical schema.`);
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
schemaId,
|
schemaId,
|
||||||
schemaName,
|
schemaName,
|
||||||
schemaVersion: isNonEmptyString(schemaVersion) ? schemaVersion : `${schemaName}.v1`,
|
schemaVersion: isNonEmptyString(schemaVersion) ? schemaVersion : `${schemaName}.v1`,
|
||||||
contentHash: isNonEmptyString(contentHash) ? contentHash : schemaId,
|
contentHash: isNonEmptyString(contentHash) ? contentHash : schemaId,
|
||||||
canonicalSchemaJson: typeof canonicalSchemaJson === "string" ? canonicalSchemaJson : "",
|
canonicalSchemaJson,
|
||||||
compatibleSchemaIds,
|
compatibleSchemaIds,
|
||||||
members: members.map(decodeCatalogMember),
|
members: members.map(decodeCatalogMember),
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
import { decode } from "@msgpack/msgpack";
|
import { decode } from "@msgpack/msgpack";
|
||||||
|
|
||||||
|
import type { CultCacheEnvelope } from "./types";
|
||||||
|
|
||||||
export const STORE_FORMAT_VERSION = "cultcache.store.v1";
|
export const STORE_FORMAT_VERSION = "cultcache.store.v1";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -151,3 +153,19 @@ export function requireStoreSlots(decoded: unknown[]): void {
|
||||||
throw new Error(`CultCache store has ${decoded.length} top-level slots; this runtime reads ${STORE_SLOTS} (header, schema catalog, records).`);
|
throw new Error(`CultCache store has ${decoded.length} top-level slots; this runtime reads ${STORE_SLOTS} (header, schema catalog, records).`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The schema id a record is stored under: the one it carries, else its type. */
|
||||||
|
export function schemaIdForEnvelope(entry: CultCacheEnvelope): string {
|
||||||
|
return entry.schemaId ?? entry.type;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* As C# WrittenOwns' declared half: the written type owns the stored schema id when its catalog entry carries that id as its own
|
||||||
|
* or lists it compatible, so a type renamed under a stable id restamps the record it already holds. (The C# half that lets an
|
||||||
|
* undeclared older id of the same schema name through when it reads whole is not ported; a same-named type keeps passing.)
|
||||||
|
* The one predicate of the key-identity rule: the store judges the file with it and the cache judges what it holds.
|
||||||
|
*/
|
||||||
|
export function writtenOwns(written: CultCacheEnvelope, storedId: string): boolean {
|
||||||
|
const entry = written.catalogEntry;
|
||||||
|
return entry !== undefined && (entry.schemaId === storedId || (entry.compatibleSchemaIds ?? []).includes(storedId));
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ import { z } from "zod";
|
||||||
import { CultCache } from "../src/cult-cache";
|
import { CultCache } from "../src/cult-cache";
|
||||||
import { inspectCultCacheBytes } from "../src/cult-cache-inspector";
|
import { inspectCultCacheBytes } from "../src/cult-cache-inspector";
|
||||||
import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
||||||
import { SingleFileMessagePackBackingStore, readStoreBytes } from "../src/single-file-messagepack-backing-store";
|
import { SingleFileMessagePackBackingStore, readSingleFileStore, readStoreBytes } from "../src/single-file-messagepack-backing-store";
|
||||||
import { SchemaConflictError, StoreUnreadableError, refuseUnreadStoreBytes } from "../src/store-format";
|
import { SchemaConflictError, StoreUnreadableError, refuseUnreadStoreBytes } from "../src/store-format";
|
||||||
import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types";
|
import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope, CultCacheSchema } from "../src/types";
|
||||||
|
|
||||||
|
|
@ -143,6 +143,63 @@ test("CultCache supports registry bootstrap, global documents, and lookup by nam
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("a put that would land a second type under a persisted key is refused, the cache and the file are unchanged, and the store reloads", async () => {
|
||||||
|
const alpha = defineDocumentType({ type: "alpha", schema: z.object({ v: z.number() }) });
|
||||||
|
const beta = defineDocumentType({ type: "beta", schema: z.object({ v: z.number() }) });
|
||||||
|
const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "cache.msgpack");
|
||||||
|
const registry = defineDocumentRegistry(alpha, beta);
|
||||||
|
const open = () => CultCache.builder().withRegistry(registry).withGenericStore(new SingleFileMessagePackBackingStore(storePath)).build();
|
||||||
|
|
||||||
|
const cache = open();
|
||||||
|
await cache.put(alpha, "shared", { v: 1 });
|
||||||
|
const before = [...(await readFile(storePath))];
|
||||||
|
await assert.rejects(() => cache.put(beta, "shared", { v: 2 }), SchemaConflictError);
|
||||||
|
assert.deepEqual([...(await readFile(storePath))], before);
|
||||||
|
assert.deepEqual(cache.get(alpha, "shared"), { v: 1 });
|
||||||
|
assert.equal(cache.get(beta, "shared"), undefined);
|
||||||
|
|
||||||
|
const reloaded = open();
|
||||||
|
await reloaded.pullAllBackingStores();
|
||||||
|
assert.deepEqual(reloaded.get(alpha, "shared"), { v: 1 });
|
||||||
|
|
||||||
|
assert.equal(await cache.delete(alpha, "shared"), true);
|
||||||
|
await cache.put(beta, "shared", { v: 2 });
|
||||||
|
const retyped = open();
|
||||||
|
await retyped.pullAllBackingStores();
|
||||||
|
assert.equal(retyped.get(alpha, "shared"), undefined);
|
||||||
|
assert.deepEqual(retyped.get(beta, "shared"), { v: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a put by a type renamed under the stored id, which lists that id compatible, restamps the record; an unrelated type is refused", async () => {
|
||||||
|
const old = defineDocumentType({ type: "ren.old", schema: z.object({ v: z.number() }), schemaId: "ren.id.v1", schemaName: "ren.old" });
|
||||||
|
const renamed = defineDocumentType({
|
||||||
|
type: "ren.new",
|
||||||
|
schema: z.object({ v: z.number() }),
|
||||||
|
schemaId: "ren.id.v2",
|
||||||
|
schemaName: "ren.new",
|
||||||
|
compatibleSchemaIds: ["ren.id.v2", "ren.id.v1"],
|
||||||
|
});
|
||||||
|
const other = defineDocumentType({ type: "ren.other", schema: z.object({ v: z.number() }), schemaId: "ren.id.other", schemaName: "ren.other" });
|
||||||
|
const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-ts-")), "cache.msgpack");
|
||||||
|
const open = (...types: (typeof old)[]) =>
|
||||||
|
types
|
||||||
|
.reduce((builder, type) => builder.withDocumentType(type), CultCache.builder())
|
||||||
|
.withGenericStore(new SingleFileMessagePackBackingStore(storePath))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
await open(old).put(old, "k", { v: 1 });
|
||||||
|
const cache = open(renamed, other as unknown as typeof old);
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
assert.deepEqual(cache.get(renamed, "k"), { v: 1 });
|
||||||
|
await assert.rejects(() => cache.put(other, "k", { v: 9 }), SchemaConflictError);
|
||||||
|
await cache.put(renamed, "k", { v: 2 });
|
||||||
|
|
||||||
|
const reloaded = open(renamed);
|
||||||
|
await reloaded.pullAllBackingStores();
|
||||||
|
assert.deepEqual(reloaded.get(renamed, "k"), { v: 2 });
|
||||||
|
assert.equal(readSingleFileStore(storePath).envelopes.filter((envelope) => envelope.key === "k").length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
test("CultCache can register name and index lookups after entries already exist", async () => {
|
test("CultCache can register name and index lookups after entries already exist", async () => {
|
||||||
const noteDocument = defineDocumentType({
|
const noteDocument = defineDocumentType({
|
||||||
type: "note",
|
type: "note",
|
||||||
|
|
@ -209,10 +266,10 @@ test("CultCache fails closed on unknown or illegal persisted polymorphic state",
|
||||||
]))
|
]))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
await assert.rejects(
|
// A record no registered type claims is carried, not a reason to refuse the load.
|
||||||
async () => cacheWithUnknownType.pullAllBackingStores(),
|
await cacheWithUnknownType.pullAllBackingStores();
|
||||||
/No schema is registered for persisted document type "forbidden"\./,
|
assert.deepEqual(cacheWithUnknownType.snapshot(), []);
|
||||||
);
|
assert.deepEqual(cacheWithUnknownType.foreignRecords().map((record) => [record.key, record.schemaName]), [["mystery", "forbidden"]]);
|
||||||
|
|
||||||
const cacheWithDuplicateGlobal = CultCache.builder()
|
const cacheWithDuplicateGlobal = CultCache.builder()
|
||||||
.withDocumentType(settingsDocument)
|
.withDocumentType(settingsDocument)
|
||||||
|
|
@ -2525,17 +2582,24 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads",
|
||||||
.filter((line) => line.length > 0 && !line.startsWith("#"))
|
.filter((line) => line.length > 0 && !line.startsWith("#"))
|
||||||
.map((line) => line.split(/\s+/u));
|
.map((line) => line.split(/\s+/u));
|
||||||
// Every vector in the folder has a manifest row.
|
// Every vector in the folder has a manifest row.
|
||||||
const listed = rows.map(([vector]) => vector!).filter((vector) => !vector.startsWith("..")).sort();
|
const listed = rows.map(([vector]) => vector!).filter((vector) => !vector.startsWith("..") && !vector.startsWith("@")).sort();
|
||||||
assert.deepEqual((await readdir(root)).filter((name) => name.endsWith(".bin")).sort(), listed);
|
assert.deepEqual((await readdir(root)).filter((name) => name.endsWith(".bin")).sort(), listed);
|
||||||
const dir = await mkdtemp(join(tmpdir(), "cultcache-readability-"));
|
const dir = await mkdtemp(join(tmpdir(), "cultcache-readability-"));
|
||||||
const seed = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "seed.msgpack");
|
const seed = await copyVector(dir, join(c2aVectors, "v3-base.msgpack"), "seed.msgpack");
|
||||||
const envelopes = await new SingleFileMessagePackBackingStore(seed).pullAll();
|
const envelopes = await new SingleFileMessagePackBackingStore(seed).pullAll();
|
||||||
for (const [vector, , , typescript] of rows) {
|
for (const [vector, , , typescript] of rows) {
|
||||||
const reads = typescript === "reads";
|
const reads = typescript === "reads";
|
||||||
const bytes = await readFile(join(root, vector));
|
// "@directory": the walker makes an empty directory at the store path; the runtime refuses to open or replace it and leaves it be.
|
||||||
|
const isDirectory = vector === "@directory";
|
||||||
|
const bytes = isDirectory ? Buffer.alloc(0) : await readFile(join(root, vector));
|
||||||
for (const operation of ["pullAll", "softPushAll", "pushAll", "push", "delete"] as const) {
|
for (const operation of ["pullAll", "softPushAll", "pushAll", "push", "delete"] as const) {
|
||||||
const file = join(dir, `${vector.replace(/[^a-z0-9.-]/giu, "_")}.${operation}.msgpack`);
|
const file = join(dir, `${vector.replace(/[^a-z0-9.-]/giu, "_")}.${operation}.msgpack`);
|
||||||
await writeFile(file, bytes);
|
if (isDirectory) {
|
||||||
|
assert.equal(typescript, "refuses", "a directory is never a store");
|
||||||
|
await mkdir(file);
|
||||||
|
} else {
|
||||||
|
await writeFile(file, bytes);
|
||||||
|
}
|
||||||
const store = new SingleFileMessagePackBackingStore(file);
|
const store = new SingleFileMessagePackBackingStore(file);
|
||||||
const run = () =>
|
const run = () =>
|
||||||
operation === "pullAll"
|
operation === "pullAll"
|
||||||
|
|
@ -2570,6 +2634,10 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads",
|
||||||
} else {
|
} else {
|
||||||
assert.ok(!keys.includes(envelopes[0]!.key), `${vector} ${operation}`);
|
assert.ok(!keys.includes(envelopes[0]!.key), `${vector} ${operation}`);
|
||||||
}
|
}
|
||||||
|
} else if (isDirectory) {
|
||||||
|
await assert.rejects(run, (error) => !(error instanceof StoreUnreadableError), `${vector} ${operation}: an I/O failure, not a typed refusal`);
|
||||||
|
assert.ok((await lstat(file)).isDirectory(), `${vector} ${operation} replaced the directory`);
|
||||||
|
assert.deepEqual(await readdir(file), [], `${vector} ${operation} wrote into the directory`);
|
||||||
} else {
|
} else {
|
||||||
await assert.rejects(
|
await assert.rejects(
|
||||||
run,
|
run,
|
||||||
|
|
@ -2582,6 +2650,27 @@ test("SingleFileMessagePackBackingStore replaces a file exactly when it reads",
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// One key is held once: a push whose type does not own the stored schema id is refused, so the writer cannot emit a store its own
|
||||||
|
// reader refuses; one whose type owns it (its own id, or one it lists compatible, as C# WrittenOwns) restamps the record in place.
|
||||||
|
test("SingleFileMessagePackBackingStore push asks whether the written type owns the stored id, not what the types are called", async () => {
|
||||||
|
// One cache holds one record per key, so each envelope is made in a cache of its own.
|
||||||
|
const owner = CultCache.builder().withDocumentType(ownsId).build();
|
||||||
|
const lister = CultCache.builder().withDocumentType(listsId).build();
|
||||||
|
await owner.put(ownsId, "k", { name: "owner" });
|
||||||
|
await lister.put(listsId, "k", { name: "lister" });
|
||||||
|
const file = join(await mkdtemp(join(tmpdir(), "cultcache-push-key-")), "store.msgpack");
|
||||||
|
const store = new SingleFileMessagePackBackingStore(file);
|
||||||
|
const idsOf = async () => ((decode(await readFile(file)) as unknown[][])[2] as unknown[][]).map((record) => [record[0], record[1]]);
|
||||||
|
|
||||||
|
await store.push(lister.getRequiredEnvelope(listsId, "k"));
|
||||||
|
await assert.rejects(() => store.push(owner.getRequiredEnvelope(ownsId, "k")), SchemaConflictError);
|
||||||
|
assert.deepEqual(await idsOf(), [["k", "id.lister"]]);
|
||||||
|
|
||||||
|
await store.pushAll([lister.getRequiredEnvelope(listsId, "k")]);
|
||||||
|
await store.push(lister.getRequiredEnvelope(listsId, "k"));
|
||||||
|
assert.equal((await store.pullAll()).length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
// Owner beats lister: a definition that owns an id and one that lists it as compatible register together in either order. A record
|
// Owner beats lister: a definition that owns an id and one that lists it as compatible register together in either order. A record
|
||||||
// under the id is the owner's, and a lister's only when nothing owns it; two listers and no owner name no single definition.
|
// under the id is the owner's, and a lister's only when nothing owns it; two listers and no owner name no single definition.
|
||||||
const ownsId = defineDocumentType({ type: "tests.owns", schema: z.object({ name: z.string() }), schemaId: "id.owned", schemaName: "tests.owns" });
|
const ownsId = defineDocumentType({ type: "tests.owns", schema: z.object({ name: z.string() }), schemaId: "id.owned", schemaName: "tests.owns" });
|
||||||
|
|
|
||||||
377
packages/cultcache-ts/test/key-identity-vector.test.ts
Normal file
377
packages/cultcache-ts/test/key-identity-vector.test.ts
Normal file
|
|
@ -0,0 +1,377 @@
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { copyFile, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import { test } from "node:test";
|
||||||
|
|
||||||
|
import { decode, encode } from "@msgpack/msgpack";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
import { CultCache } from "../src/cult-cache";
|
||||||
|
import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
||||||
|
import { SingleFileMessagePackBackingStore, readSingleFileStore } from "../src/single-file-messagepack-backing-store";
|
||||||
|
import { SchemaConflictError } from "../src/store-format";
|
||||||
|
import type { AnyCultCacheDocumentDefinition, CultCacheEnvelope } from "../src/types";
|
||||||
|
|
||||||
|
// variants:ruling:cache-refuses-retype, write-ownership-by-written-type, unheld-removal-refused-at-commit. The vectors in
|
||||||
|
// tests/vectors/key-identity are written by the C# reference (see its README for the rule and the row shapes); this walks
|
||||||
|
// every row against the TS cache. A leaf keeps no verdict of its own.
|
||||||
|
|
||||||
|
function findVectors(from: string): string {
|
||||||
|
for (let dir = from; ; dir = resolve(dir, "..")) {
|
||||||
|
if (existsSync(join(dir, "tests", "vectors", "key-identity"))) return join(dir, "tests", "vectors", "key-identity");
|
||||||
|
if (resolve(dir, "..") === dir) throw new Error("tests/vectors/key-identity not found above the test.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const vectors = findVectors(__dirname);
|
||||||
|
|
||||||
|
// One string slot [0: Text]; the schema ids are the ones C# wrote (README). The type is the schema name, the name a record
|
||||||
|
// read back from a store carries as its type (the store's delete matches on it).
|
||||||
|
function kiType(type: string, id: string) {
|
||||||
|
const schemaName = `tests.ki_${type}`;
|
||||||
|
return defineDocumentType({
|
||||||
|
type: schemaName,
|
||||||
|
schemaId: `sha256:${id}`,
|
||||||
|
schemaName,
|
||||||
|
schemaVersion: `${schemaName}.v1`,
|
||||||
|
schema: z.object({ text: z.string() }),
|
||||||
|
formatter: {
|
||||||
|
encode: (value: { text: string }) => encode([value.text]),
|
||||||
|
decode: (payload: Uint8Array) => ({ text: (decode(payload) as [string])[0] }),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const types: Record<string, AnyCultCacheDocumentDefinition> = {
|
||||||
|
alpha: kiType("alpha", "b20703a98f49c253898404ab50ba2c630720f0b8a5e1a41614d5650be22799fe"),
|
||||||
|
beta: kiType("beta", "dee4b727cf2ee04e61ab635ff0c5f3dca7345a411dc5f1018e583ab16009cf94"),
|
||||||
|
gamma: kiType("gamma", "fbc26e351e00c9e7185e8de171c7486aba707c229c949f0aae6b889288af5f6d"),
|
||||||
|
};
|
||||||
|
|
||||||
|
type Op = [string, ...unknown[]];
|
||||||
|
type Row = [string, string[], Op[], [[string] | null, [string, string, string][], [string, string][] | null, string[], string[]]];
|
||||||
|
|
||||||
|
const compare = (left: string, right: string) => (left < right ? -1 : left > right ? 1 : 0);
|
||||||
|
const textOf = (envelope: CultCacheEnvelope) => (decode(documentPayloadBytes(envelope.payload)) as [string])[0];
|
||||||
|
const stored = (path: string) => new Map(readSingleFileStore(path).envelopes.map((e) => [e.key, e]));
|
||||||
|
const sameRecord = (left: CultCacheEnvelope | undefined, right: CultCacheEnvelope | undefined) =>
|
||||||
|
left !== undefined && right !== undefined && left.schemaId === right.schemaId && left.storedAt === right.storedAt &&
|
||||||
|
Buffer.compare(Buffer.from(documentPayloadBytes(left.payload)), Buffer.from(documentPayloadBytes(right.payload))) === 0;
|
||||||
|
|
||||||
|
test("vectors rows.msgpack: every row's refusal, held records, file, foreign list and kept bytes", async () => {
|
||||||
|
const rows = decode(await readFile(join(vectors, "rows.msgpack"))) as Row[];
|
||||||
|
assert.equal(rows.length, 12);
|
||||||
|
let walked = 0;
|
||||||
|
const failures: string[] = [];
|
||||||
|
for (const [label, registered, ops, expect] of rows) {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "ki-"));
|
||||||
|
try {
|
||||||
|
const [refusal, held, file, foreign, kept] = expect;
|
||||||
|
let counter = 0;
|
||||||
|
let path: string | undefined;
|
||||||
|
let cache!: CultCache;
|
||||||
|
let before: Map<string, CultCacheEnvelope> | undefined;
|
||||||
|
const attached: string[] = [];
|
||||||
|
const build = (store: string | undefined) => {
|
||||||
|
let builder = CultCache.builder();
|
||||||
|
for (const name of registered) builder = builder.withDocumentType(types[name]!);
|
||||||
|
if (store !== undefined) builder = builder.withGenericStore(new SingleFileMessagePackBackingStore(store));
|
||||||
|
return builder.build();
|
||||||
|
};
|
||||||
|
const fresh = async (name: string) => {
|
||||||
|
const copy = join(dir, `${counter++}-${name}`);
|
||||||
|
await copyFile(join(vectors, name), copy);
|
||||||
|
return copy;
|
||||||
|
};
|
||||||
|
|
||||||
|
let refused: unknown;
|
||||||
|
let failed = false;
|
||||||
|
let beforeBytes: Buffer | undefined;
|
||||||
|
let beforeCache: CultCacheEnvelope[] = [];
|
||||||
|
let beforeAttached: Buffer[] = [];
|
||||||
|
for (const [index, op] of ops.entries()) {
|
||||||
|
const last = index === ops.length - 1;
|
||||||
|
if (last) {
|
||||||
|
beforeCache = cache?.snapshot() ?? [];
|
||||||
|
beforeAttached = await Promise.all(attached.map((copy) => readFile(copy)));
|
||||||
|
beforeBytes = path !== undefined ? await readFile(path) : undefined;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
switch (op[0]) {
|
||||||
|
case "open": {
|
||||||
|
const arg = op[1] as string | null;
|
||||||
|
path = arg === "memory" ? undefined : arg === null ? join(dir, "fresh.cc") : await fresh(arg);
|
||||||
|
if (path !== undefined && arg !== null) before = stored(path);
|
||||||
|
cache = build(path);
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "attach": {
|
||||||
|
const copy = await fresh(op[1] as string);
|
||||||
|
attached.push(copy);
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(copy), ...(op[2] as string[]).map((name) => types[name]!));
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "reopen":
|
||||||
|
cache = build(path);
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
break;
|
||||||
|
case "put":
|
||||||
|
await cache.put(types[op[1] as string]!, op[2] as string, { text: op[3] as string } as never);
|
||||||
|
break;
|
||||||
|
case "delete":
|
||||||
|
await cache.delete(types[op[1] as string]!, op[2] as string);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
assert.fail(`${label}: unknown op ${op[0]}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (!last) throw error;
|
||||||
|
refused = error;
|
||||||
|
failed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (refusal === null) {
|
||||||
|
assert.equal(failed, false, `${label}: the last op lands (${String(refused)})`);
|
||||||
|
} else {
|
||||||
|
assert.equal(failed, true, `${label}: the last op is refused`);
|
||||||
|
const [key] = refusal;
|
||||||
|
if (label === "key-in-two-stores-refused") {
|
||||||
|
// The key is read out of the refusal as a delimited token: the message opens "Record <key> is held by" and the key
|
||||||
|
// occurs nowhere else in it (not inside a schema name).
|
||||||
|
const message = (refused as Error).message;
|
||||||
|
const named = /^Record (\S+) is held by/u.exec(message);
|
||||||
|
assert.equal(named?.[1], key, `${label}: the refusal opens with the key`);
|
||||||
|
assert.equal(message.split(/[^A-Za-z0-9_.]+/u).filter((token) => token === key).length, 1, `${label}: the key occurs only as a delimited token`);
|
||||||
|
assert.deepEqual(cache.snapshot(), beforeCache, `${label}: the earlier pulled state survives`);
|
||||||
|
assert.deepEqual(await Promise.all(attached.slice(0, beforeAttached.length).map((copy) => readFile(copy))), beforeAttached, `${label}: the attached files are byte-identical`);
|
||||||
|
} else {
|
||||||
|
assert.ok(refused instanceof SchemaConflictError, `${label}: a typed schema conflict, got ${String(refused)}`);
|
||||||
|
assert.equal((refused as SchemaConflictError).recordKey, key, `${label}: names the key`);
|
||||||
|
assert.deepEqual(cache.snapshot(), beforeCache, `${label}: the cache is unchanged`);
|
||||||
|
if (path !== undefined) assert.deepEqual(await readFile(path), beforeBytes, `${label}: the store is byte-identical`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
cache.snapshot()
|
||||||
|
.map((e) => [e.type.replace("tests.ki_", ""), e.key, textOf(e)] as [string, string, string])
|
||||||
|
.sort((l, r) => compare(l[0], r[0]) || compare(l[1], r[1])),
|
||||||
|
held,
|
||||||
|
`${label}: held`,
|
||||||
|
);
|
||||||
|
if (path === undefined) {
|
||||||
|
assert.equal(file, null, `${label}: no store`);
|
||||||
|
} else {
|
||||||
|
assert.deepEqual(
|
||||||
|
[...stored(path).values()]
|
||||||
|
.map((e) => [e.key, e.catalogEntry?.schemaName ?? e.type] as [string, string])
|
||||||
|
.sort((l, r) => compare(l[0], r[0])),
|
||||||
|
file,
|
||||||
|
`${label}: file`,
|
||||||
|
);
|
||||||
|
if (before !== undefined) {
|
||||||
|
const after = stored(path);
|
||||||
|
assert.deepEqual(foreign.filter((key) => sameRecord(before!.get(key), after.get(key))), kept, `${label}: kept`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.deepEqual(cache.foreignRecords().map((record: { key: string }) => record.key), foreign, `${label}: foreign`);
|
||||||
|
walked++;
|
||||||
|
} catch (error) {
|
||||||
|
failures.push(`${label}: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.deepEqual(failures, [], "every row holds its verdict");
|
||||||
|
assert.equal(walked, 12);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Beyond the vector: what the walker's string-typed rows cannot say.
|
||||||
|
const CANARY = "CANARY-ki-77c1";
|
||||||
|
const plain = (type: string, schemaId: string, compatibleSchemaIds?: string[]) =>
|
||||||
|
defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: z.object({ v: z.string() }), ...(compatibleSchemaIds ? { compatibleSchemaIds } : {}) });
|
||||||
|
|
||||||
|
test("a refusal names the key and schemas and never echoes the written value", async () => {
|
||||||
|
const alpha = plain("cn.alpha", "cn.alpha.v1");
|
||||||
|
const beta = plain("cn.beta", "cn.beta.v1");
|
||||||
|
const cache = CultCache.builder().withDocumentType(alpha).withDocumentType(beta).build();
|
||||||
|
await cache.put(alpha, "k", { v: "first" });
|
||||||
|
await assert.rejects(cache.put(beta, "k", { v: CANARY }), (error: Error) => {
|
||||||
|
assert.ok(error instanceof SchemaConflictError && error.recordKey === "k");
|
||||||
|
assert.ok(!error.message.includes(CANARY));
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
assert.deepEqual(cache.get(alpha, "k"), { v: "first" });
|
||||||
|
assert.equal(cache.get(beta, "k"), undefined);
|
||||||
|
await assert.rejects(cache.putEnvelope(beta, { key: "k", type: "cn.beta", payload: documentPayloadFromBytes(encode({ v: CANARY })), storedAt: "2026-01-01T00:00:00Z" }), SchemaConflictError);
|
||||||
|
assert.equal(cache.snapshot().length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a type that owns the held record's id replaces it: one record at the key, no second type held", async () => {
|
||||||
|
const old = plain("cn.old", "cn.id.v1");
|
||||||
|
const renamed = plain("cn.new", "cn.id.v2", ["cn.id.v2", "cn.id.v1"]);
|
||||||
|
const cache = CultCache.builder().withDocumentType(old).withDocumentType(renamed).build();
|
||||||
|
await cache.put(old, "k", { v: "a" });
|
||||||
|
await cache.put(renamed, "k", { v: "b" });
|
||||||
|
assert.deepEqual(cache.snapshot().map((e) => [e.type, e.key]), [["cn.new", "k"]]);
|
||||||
|
assert.equal(cache.get(old, "k"), undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a store carrying an unregistered record loads, lists it sorted, keeps it through writes of other keys, and a later registration claims it", async () => {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "ki-"));
|
||||||
|
try {
|
||||||
|
const path = join(dir, "s.cc");
|
||||||
|
await copyFile(join(vectors, "foreign-store.cc"), path);
|
||||||
|
const open = (...names: string[]) => {
|
||||||
|
let builder = CultCache.builder();
|
||||||
|
for (const name of names) builder = builder.withDocumentType(types[name]!);
|
||||||
|
return builder.withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||||
|
};
|
||||||
|
const cache = open("alpha");
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
const record = stored(path).get("k")!;
|
||||||
|
assert.deepEqual(cache.foreignRecords(), [
|
||||||
|
{ key: "k", schemaId: "sha256:fbc26e351e00c9e7185e8de171c7486aba707c229c949f0aae6b889288af5f6d", schemaName: "tests.ki_gamma", storedAt: record.storedAt },
|
||||||
|
]);
|
||||||
|
await cache.put(types.alpha!, "z", { text: "z" } as never);
|
||||||
|
await cache.delete(types.alpha!, "j");
|
||||||
|
assert.ok(sameRecord(stored(path).get("k"), record), "the foreign record's stored bytes survive");
|
||||||
|
assert.equal(await cache.delete(types.alpha!, "nobody"), false);
|
||||||
|
|
||||||
|
const claimed = open("alpha", "gamma");
|
||||||
|
await claimed.pullAllBackingStores();
|
||||||
|
assert.deepEqual(claimed.foreignRecords(), []);
|
||||||
|
assert.equal(claimed.snapshot().length, 2);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a foreign key one store carries and another holds under a registered type refuses the load and admits nothing", async () => {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "ki-"));
|
||||||
|
try {
|
||||||
|
const foreign = join(dir, "foreign.cc");
|
||||||
|
const catalog = join(dir, "catalog.cc");
|
||||||
|
await copyFile(join(vectors, "catalog-store.cc"), catalog);
|
||||||
|
const writer = CultCache.builder().withDocumentType(types.gamma!).withGenericStore(new SingleFileMessagePackBackingStore(foreign)).build();
|
||||||
|
await writer.put(types.gamma!, "qx", { text: "g" } as never);
|
||||||
|
const cache = CultCache.builder().withDocumentType(types.alpha!).build();
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(catalog), types.alpha!);
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(foreign));
|
||||||
|
await assert.rejects(cache.pullAllBackingStores(), /held by two stores/u);
|
||||||
|
assert.deepEqual(cache.snapshot().map((e) => e.key), ["qx"], "the earlier pulled state survives");
|
||||||
|
assert.deepEqual(cache.foreignRecords(), []);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// The cache judge answers for what the cache holds whatever the store is: a store that never judges (it only records
|
||||||
|
// what it is asked) is asked nothing at a foreign key, and foreign records list by key ordinal.
|
||||||
|
test("a foreign key is refused by the cache before any store is asked, and foreign records list sorted", async () => {
|
||||||
|
const calls: string[] = [];
|
||||||
|
const record = (key: string, schemaId: string): CultCacheEnvelope => ({
|
||||||
|
key,
|
||||||
|
type: "unclaimed",
|
||||||
|
schemaId,
|
||||||
|
storedAt: "2026-01-01T00:00:00Z",
|
||||||
|
payload: documentPayloadFromBytes(encode(["x"])),
|
||||||
|
});
|
||||||
|
const store = {
|
||||||
|
pullAll: async () => [record("m", "unclaimed.m.v1"), record("b", "unclaimed.b.v1")],
|
||||||
|
push: async (entry: CultCacheEnvelope) => void calls.push(`push ${entry.key}`),
|
||||||
|
delete: async (entry: CultCacheEnvelope) => void calls.push(`delete ${entry.key}`),
|
||||||
|
};
|
||||||
|
const cache = CultCache.builder().withDocumentType(types.alpha!).withGenericStore(store).build();
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
assert.deepEqual(cache.foreignRecords().map((r) => [r.key, r.schemaId, r.schemaName]), [["b", "unclaimed.b.v1", "unclaimed"], ["m", "unclaimed.m.v1", "unclaimed"]]);
|
||||||
|
await assert.rejects(cache.put(types.alpha!, "m", { text: CANARY } as never), (error: Error) => {
|
||||||
|
assert.ok(error instanceof SchemaConflictError && error.recordKey === "m");
|
||||||
|
assert.ok(!error.message.includes(CANARY));
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
await assert.rejects(cache.delete(types.alpha!, "b"), (error: Error) => error instanceof SchemaConflictError && error.recordKey === "b");
|
||||||
|
assert.equal(await cache.delete(types.alpha!, "nobody"), false);
|
||||||
|
assert.deepEqual(calls, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
// The store's delete matches the record the file stores by its schema id, not by the type string: a record read back is
|
||||||
|
// typed by its schema's name, which a type of another name does not carry.
|
||||||
|
test("a delete removes the record the file stores although the type differs from the schema name, and the key retypes across a reload", async () => {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "ki-"));
|
||||||
|
try {
|
||||||
|
const path = join(dir, "s.cc");
|
||||||
|
const named = (type: string, schemaName: string, schemaId: string) =>
|
||||||
|
defineDocumentType({ type, schemaId, schemaName, schemaVersion: `${schemaName}.v1`, schema: z.object({ v: z.string() }) });
|
||||||
|
const first = named("app.first", "wire.first", "wire.first.v1");
|
||||||
|
const second = named("app.second", "wire.second", "wire.second.v1");
|
||||||
|
const open = () => CultCache.builder().withDocumentType(first).withDocumentType(second).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||||
|
const writer = open();
|
||||||
|
await writer.put(first, "k", { v: "a" });
|
||||||
|
const reader = open();
|
||||||
|
await reader.pullAllBackingStores();
|
||||||
|
assert.equal(await reader.delete(first, "k"), true);
|
||||||
|
assert.deepEqual([...stored(path).keys()], [], "the record is gone from the file");
|
||||||
|
await reader.put(second, "k", { v: "b" });
|
||||||
|
assert.deepEqual([...stored(path).values()].map((e) => [e.key, e.catalogEntry?.schemaName]), [["k", "wire.second"]]);
|
||||||
|
const again = open();
|
||||||
|
await again.pullAllBackingStores();
|
||||||
|
assert.deepEqual(again.snapshot().map((e) => [e.type, e.key]), [["app.second", "k"]]);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an owned rename whose two types home in different stores is refused, and neither store changes", async () => {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "ki-"));
|
||||||
|
try {
|
||||||
|
const oldPath = join(dir, "old.cc");
|
||||||
|
const newPath = join(dir, "new.cc");
|
||||||
|
const old = plain("rn.old", "rn.id.v1");
|
||||||
|
const renamed = plain("rn.new", "rn.id.v2", ["rn.id.v2", "rn.id.v1"]);
|
||||||
|
const cache = CultCache.builder().withDocumentType(old).withDocumentType(renamed).build();
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(oldPath), old);
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(newPath), renamed);
|
||||||
|
await cache.put(old, "k", { v: "a" });
|
||||||
|
const before = await readFile(oldPath);
|
||||||
|
await assert.rejects(cache.put(renamed, "k", { v: CANARY }), (error: Error) => {
|
||||||
|
assert.ok(error instanceof SchemaConflictError && error.recordKey === "k");
|
||||||
|
assert.ok(!error.message.includes(CANARY));
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
assert.deepEqual(await readFile(oldPath), before);
|
||||||
|
assert.equal(existsSync(newPath), false, "the new home was never written");
|
||||||
|
assert.deepEqual(cache.snapshot().map((e) => [e.type, e.key]), [["rn.old", "k"]]);
|
||||||
|
// Same home: the rename replaces.
|
||||||
|
const together = CultCache.builder().withDocumentType(old).withDocumentType(renamed).withGenericStore(new SingleFileMessagePackBackingStore(join(dir, "one.cc"))).build();
|
||||||
|
await together.put(old, "k", { v: "a" });
|
||||||
|
await together.put(renamed, "k", { v: "b" });
|
||||||
|
assert.deepEqual(together.snapshot().map((e) => [e.type, e.key]), [["rn.new", "k"]]);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("two stores each carrying an unregistered record at one key refuse the load and admit nothing", async () => {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "ki-"));
|
||||||
|
try {
|
||||||
|
const paths = [join(dir, "a.cc"), join(dir, "b.cc")];
|
||||||
|
for (const path of paths) {
|
||||||
|
const writer = CultCache.builder().withDocumentType(types.gamma!).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||||
|
await writer.put(types.gamma!, "k", { text: "g" } as never);
|
||||||
|
}
|
||||||
|
// Neither store's records are claimed: gamma is not registered here.
|
||||||
|
const cache = CultCache.builder().withDocumentType(types.alpha!).withDocumentType(types.beta!).build();
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(paths[0]!), types.alpha!);
|
||||||
|
await cache.addBackingStore(new SingleFileMessagePackBackingStore(paths[1]!), types.beta!);
|
||||||
|
await assert.rejects(cache.pullAllBackingStores(), /^Error: Record k is held by two stores/u);
|
||||||
|
assert.deepEqual(cache.foreignRecords(), []);
|
||||||
|
assert.deepEqual(cache.snapshot(), []);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
@ -12,6 +12,7 @@ import { decode, encode } from "@msgpack/msgpack";
|
||||||
|
|
||||||
import { documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
import { documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
||||||
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
|
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
|
||||||
|
import { SchemaConflictError, StoreUnreadableError } from "../src/store-format";
|
||||||
import type { CultCacheEnvelope } from "../src/types";
|
import type { CultCacheEnvelope } from "../src/types";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
|
@ -571,15 +572,84 @@ const snapshotOf = async (store: SingleFileMessagePackBackingStore): Promise<str
|
||||||
.map((entry) => `${entry.type}/${entry.key}:${[...documentPayloadBytes(entry.payload)].at(-1)}`)
|
.map((entry) => `${entry.type}/${entry.key}:${[...documentPayloadBytes(entry.payload)].at(-1)}`)
|
||||||
.sort();
|
.sort();
|
||||||
|
|
||||||
test("push replaces the record at the same type and key and keeps every other", async () => {
|
test("push replaces the record under the same key and type and keeps every other; retyping is delete then push", async () => {
|
||||||
const directory = await scratch();
|
const directory = await scratch();
|
||||||
try {
|
try {
|
||||||
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
||||||
await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]);
|
await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("c", "u", 3)]);
|
||||||
await store.push(entryOf("a", "t", 9));
|
await store.push(entryOf("a", "t", 9));
|
||||||
assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/a:3"]);
|
assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "u/c:3"]);
|
||||||
await store.push(entryOf("c", "t", 4));
|
await store.push(entryOf("d", "t", 4));
|
||||||
assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/c:4", "u/a:3"]);
|
assert.deepEqual(await snapshotOf(store), ["t/a:9", "t/b:2", "t/d:4", "u/c:3"]);
|
||||||
|
await store.delete(entryOf("a", "t"));
|
||||||
|
await store.push(entryOf("a", "u", 5));
|
||||||
|
assert.deepEqual(await snapshotOf(store), ["t/b:2", "t/d:4", "u/a:5", "u/c:3"]);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a push that would land a second type under an existing key is refused and the file is byte for byte unchanged", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "store.cc");
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
await store.pushAll([entryOf("k", "t", 1), entryOf("m", "t", 2)]);
|
||||||
|
const before = [...(await readFile(path))];
|
||||||
|
await assert.rejects(
|
||||||
|
() => store.push(entryOf("k", "u", 9)),
|
||||||
|
(error: unknown) => error instanceof SchemaConflictError && error.recordKey === "k" && /holds record "k" as type "t"/u.test(error.message),
|
||||||
|
);
|
||||||
|
assert.deepEqual([...(await readFile(path))], before);
|
||||||
|
assert.deepEqual(await snapshotOf(new SingleFileMessagePackBackingStore(path)), ["t/k:1", "t/m:2"]);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a pushAll batch that carries one key under two types is refused and writes nothing", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "store.cc");
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
await assert.rejects(() => store.pushAll([entryOf("k", "t", 1), entryOf("k", "u", 2)]), SchemaConflictError);
|
||||||
|
assert.equal((await readdir(directory)).includes("store.cc"), false);
|
||||||
|
await store.pushAll([entryOf("a", "t", 1)]);
|
||||||
|
const before = [...(await readFile(path))];
|
||||||
|
await assert.rejects(() => store.pushAll([entryOf("k", "t", 1), entryOf("k", "u", 2)]), SchemaConflictError);
|
||||||
|
assert.deepEqual([...(await readFile(path))], before);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a pushAll batch that carries one key twice under one type keeps the last and the file reads back with one record", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "store.cc");
|
||||||
|
await new SingleFileMessagePackBackingStore(path).pushAll([entryOf("k", "t", 1), entryOf("a", "t", 3), entryOf("k", "t", 2)]);
|
||||||
|
assert.deepEqual(await snapshotOf(new SingleFileMessagePackBackingStore(path)), ["t/a:3", "t/k:2"]);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a legacy envelope array that holds a key under two types is refused on open, as a v1 store is", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "legacy.cc");
|
||||||
|
const bytes = encode([
|
||||||
|
{ key: "a", type: "t", payload: { n: 1 }, storedAt: "2026-10-20T00:00:00Z" },
|
||||||
|
{ key: "a", type: "u", payload: { n: 2 }, storedAt: "2026-10-20T00:00:00Z" },
|
||||||
|
]);
|
||||||
|
await writeFile(path, bytes);
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
await assert.rejects(
|
||||||
|
() => store.pullAll(),
|
||||||
|
(error: unknown) => error instanceof StoreUnreadableError && /"a" appears more than once/u.test(error.message),
|
||||||
|
);
|
||||||
|
await assert.rejects(() => store.push(entryOf("b")), StoreUnreadableError);
|
||||||
|
assert.deepEqual([...(await readFile(path))], [...bytes]);
|
||||||
} finally {
|
} finally {
|
||||||
await rm(directory, { recursive: true, force: true });
|
await rm(directory, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
|
|
@ -589,11 +659,12 @@ test("delete removes only the record at that type and key", async () => {
|
||||||
const directory = await scratch();
|
const directory = await scratch();
|
||||||
try {
|
try {
|
||||||
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
||||||
await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("a", "u", 3)]);
|
await store.pushAll([entryOf("a", "t", 1), entryOf("b", "t", 2), entryOf("c", "u", 3)]);
|
||||||
await store.delete(entryOf("a", "t"));
|
await store.delete(entryOf("a", "t"));
|
||||||
assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]);
|
assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/c:3"]);
|
||||||
await store.delete(entryOf("zzz", "t"));
|
await store.delete(entryOf("zzz", "t"));
|
||||||
assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/a:3"]);
|
await store.delete(entryOf("c", "t"));
|
||||||
|
assert.deepEqual(await snapshotOf(store), ["t/b:2", "u/c:3"]);
|
||||||
} finally {
|
} finally {
|
||||||
await rm(directory, { recursive: true, force: true });
|
await rm(directory, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -119,14 +119,6 @@ test("a removed or changed holder releases its value", async () => {
|
||||||
assert.equal(cache.getKeyByIndex(document, "sku", "T"), "a");
|
assert.equal(cache.getKeyByIndex(document, "sku", "T"), "a");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the keys in a refusal are in ordinal order whichever one lands second", async () => {
|
|
||||||
const document = item("item");
|
|
||||||
const cache = memory(document);
|
|
||||||
await cache.put(document, "m", { sku: "S" });
|
|
||||||
await assert.rejects(cache.put(document, "z", { sku: "S" }), { message: refusal("m", "z", "tests.item", "sku") });
|
|
||||||
await assert.rejects(cache.put(document, "B", { sku: "S" }), { message: refusal("B", "m", "tests.item", "sku") });
|
|
||||||
});
|
|
||||||
|
|
||||||
// The C# White_Space set, and nothing else, is blank. Every UTF-16 code unit is judged, so a rule written for a few
|
// The C# White_Space set, and nothing else, is blank. Every UTF-16 code unit is judged, so a rule written for a few
|
||||||
// characters (or for JavaScript's \s, which counts U+FEFF and omits U+0085) cannot pass.
|
// characters (or for JavaScript's \s, which counts U+FEFF and omits U+0085) cannot pass.
|
||||||
const CSHARP_WHITE_SPACE = (code: number) =>
|
const CSHARP_WHITE_SPACE = (code: number) =>
|
||||||
|
|
@ -365,6 +357,77 @@ test("vectors duplicate-store.cc and blank-store.cc hold their verdicts", async
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// refusals.msgpack: rows [label, ops, refusal] written by the C# reference. TypeScript has no batch commit
|
||||||
|
// (variants:idea:unique-index-runtimes.ts-batch-commit), so it performs every row whose commits each hold one record:
|
||||||
|
// 5, 8-13, 19-22, 24, 25, 27, 29, 31, 32, 33, 35 and 37 of the README (20).
|
||||||
|
const REFUSAL_SCHEMAS: [string, string, string, Record<string, "sku" | "second">][] = [
|
||||||
|
["unique_item", "tests.unique_item", "sha256:93bb890de83069061af696b23f94fd32fb3c2dad3c91ee645e8add56085f1882", { Sku: "sku" }],
|
||||||
|
["uq_pair", "tests.uq_pair", "sha256:2538cceafb6dcbce22ef11b932ecb804af5f96bc3aaedff692a52d3640433e60", { Sku: "sku", Alt: "second" }],
|
||||||
|
["uq_alias", "tests.uq_alias", "sha256:77a77a394ebd8bf71e14b9b6f2cea4978c25d4e705fd12364f660edbf6c63b1f", { Sku: "sku", alt: "second" }],
|
||||||
|
["uq_wide", "tests.uq_wide", "sha256:b79eecd11eed5e456902f095d2748da7f59125c4c8972b0da7dddc81d07fc03f", { "\uFF21": "sku", "\u{10400}": "second" }],
|
||||||
|
["uq_alpha", "tests.uq_alpha", "sha256:53aeaf2ea9b25e85ade6b15fc04c7653e7b206c87e258f1aa74dd32a840a1389", { Sku: "sku" }],
|
||||||
|
["uq_beta", "tests.uq_beta", "sha256:45ed62a64c91f6d815d3ec975c4a4a1b2ce07b6e7262a32d828673cf10899586", { Sku: "sku" }],
|
||||||
|
];
|
||||||
|
|
||||||
|
type RefusalOp = ["open", string | null] | ["pull", string] | ["commit", [string, string, string, string][]];
|
||||||
|
|
||||||
|
test("vectors refusals.msgpack: every row TS can perform", async () => {
|
||||||
|
const rows = decode(await readFile(join(vectors, "refusals.msgpack"))) as [string, RefusalOp[], [string, string, string, string] | null][];
|
||||||
|
const definitions = new Map(
|
||||||
|
REFUSAL_SCHEMAS.map(([type, schemaName, schemaId, indexes]) => [
|
||||||
|
schemaName,
|
||||||
|
defineDocumentType({
|
||||||
|
type,
|
||||||
|
schemaId,
|
||||||
|
schemaName,
|
||||||
|
schemaVersion: `${schemaName}.v1`,
|
||||||
|
schema: z.object({ sku: z.string(), second: z.string() }),
|
||||||
|
codec: codecs.slots<{ sku: string; second: string }>({ 0: ["sku", codecs.string], 1: ["second", codecs.string] }, `Uq${type}`),
|
||||||
|
indexes,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "uidx-refusals-"));
|
||||||
|
let walked = 0;
|
||||||
|
try {
|
||||||
|
for (const [label, ops, refused] of rows) {
|
||||||
|
if (ops.some((op) => op[0] === "commit" && op[1].length > 1)) continue;
|
||||||
|
const file = join(dir, `${walked}.cc`);
|
||||||
|
let cache: CultCache | undefined;
|
||||||
|
const perform = async (op: RefusalOp) => {
|
||||||
|
if (op[0] === "open") {
|
||||||
|
if (op[1] !== null) await copyFile(join(vectors, op[1]), file);
|
||||||
|
let builder = CultCache.builder().withGenericStore(new SingleFileMessagePackBackingStore(file));
|
||||||
|
for (const definition of definitions.values()) builder = builder.withDocumentType(definition);
|
||||||
|
cache = builder.build();
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
} else if (op[0] === "pull") {
|
||||||
|
await copyFile(join(vectors, op[1]), file);
|
||||||
|
await cache!.pullAllBackingStores();
|
||||||
|
} else {
|
||||||
|
for (const [key, schema, sku, second] of op[1]) await cache!.put(definitions.get(schema)!, key, { sku, second });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for (const op of ops.slice(0, -1)) await perform(op);
|
||||||
|
const last = ops[ops.length - 1];
|
||||||
|
if (refused === null) {
|
||||||
|
await perform(last);
|
||||||
|
} else {
|
||||||
|
const [k1, k2, schema, alias] = refused;
|
||||||
|
const before = last[0] === "commit" || (last[0] === "open" && last[1] === null)
|
||||||
|
? (existsSync(file) ? await readFile(file) : Buffer.alloc(0))
|
||||||
|
: await readFile(join(vectors, last[1]!));
|
||||||
|
await assert.rejects(perform(last), { message: refusal(k1, k2, schema, alias) }, label);
|
||||||
|
assert.deepEqual(existsSync(file) ? await readFile(file) : Buffer.alloc(0), before, `${label}: the file is untouched`);
|
||||||
|
}
|
||||||
|
walked++;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
assert.equal(walked, 20);
|
||||||
|
});
|
||||||
|
|
||||||
test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => {
|
test("a refused load leaves the cache's prior state as it was, and a later good load still works", async () => {
|
||||||
const dir = await mkdtemp(join(tmpdir(), "uidx-load-"));
|
const dir = await mkdtemp(join(tmpdir(), "uidx-load-"));
|
||||||
try {
|
try {
|
||||||
|
|
@ -381,11 +444,21 @@ test("a refused load leaves the cache's prior state as it was, and a later good
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test("registerIndex over held duplicates installs nothing", async () => {
|
test("registerIndex refuses a name the definition declares as a role", async () => {
|
||||||
const document = item("item", "roles");
|
const document = item("item", "roles");
|
||||||
const cache = memory(document);
|
const cache = memory(document);
|
||||||
await cache.put(document, "a", { sku: "S", group: "g" });
|
await cache.put(document, "a", { sku: "S", group: "g" });
|
||||||
await cache.put(document, "b", { sku: "S", group: "h" });
|
await cache.put(document, "b", { sku: "S", group: "h" });
|
||||||
|
await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: /declared as a role; a role is never a unique index/u });
|
||||||
|
assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined);
|
||||||
|
await cache.put(document, "c", { sku: "S", group: "i" });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("registerIndex over held duplicates installs nothing", async () => {
|
||||||
|
const document = item("item", "none");
|
||||||
|
const cache = memory(document);
|
||||||
|
await cache.put(document, "a", { sku: "S", group: "g" });
|
||||||
|
await cache.put(document, "b", { sku: "S", group: "h" });
|
||||||
await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: refusal("a", "b", "tests.item", "sku") });
|
await assert.rejects(cache.registerIndex(document, "sku", "sku"), { message: refusal("a", "b", "tests.item", "sku") });
|
||||||
assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined, "no map was installed");
|
assert.equal(cache.getKeyByIndex(document, "sku", "S"), undefined, "no map was installed");
|
||||||
await cache.put(document, "c", { sku: "S", group: "i" });
|
await cache.put(document, "c", { sku: "S", group: "i" });
|
||||||
|
|
@ -468,7 +541,7 @@ test("a load judges every unique alias: a store collision on the second alias re
|
||||||
});
|
});
|
||||||
|
|
||||||
test("registerIndex of a second unique alias judges it, naming the two smallest keys whatever order they were written in", async () => {
|
test("registerIndex of a second unique alias judges it, naming the two smallest keys whatever order they were written in", async () => {
|
||||||
const document = pair({ indexes: { a: "a" }, roles: { b: "b" } });
|
const document = pair({ indexes: { a: "a" } });
|
||||||
const cache = memory(document);
|
const cache = memory(document);
|
||||||
await cache.put(document, "z", { a: "1", b: "s" });
|
await cache.put(document, "z", { a: "1", b: "s" });
|
||||||
await cache.put(document, "m", { a: "2", b: "s" });
|
await cache.put(document, "m", { a: "2", b: "s" });
|
||||||
|
|
@ -478,27 +551,33 @@ test("registerIndex of a second unique alias judges it, naming the two smallest
|
||||||
assert.equal(cache.getKeyByIndex(document, "b", "s"), undefined);
|
assert.equal(cache.getKeyByIndex(document, "b", "s"), undefined);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("verdict parity: the same stores are refused whatever the load order, and the first collision refuses (text is informational)", async () => {
|
// ---- the single-put judge's two orders, which no pair-order vector row reaches from TS (a vector row's second record
|
||||||
const dir = await mkdtemp(join(tmpdir(), "uidx-verdict-"));
|
// arrives in a batch that TS cannot perform): aliases by UTF-16 code unit, then the key pair by UTF-16 code unit ----
|
||||||
try {
|
|
||||||
// Three records, one pair colliding; whichever position the pair has in the file, the store is refused and a clean store opens.
|
test("a put colliding on two aliases names the one least by UTF-16 code unit, not by code point", async () => {
|
||||||
const roles = pair({ roles: { a: "a" } });
|
// U+10400 is the surrogate pair D801 DC00, so it sorts before U+FF21 by code unit and after it by code point.
|
||||||
for (const [name, values, refused] of [
|
const document = pair({ indexes: { "A": "a", "\u{10400}": "b" } });
|
||||||
["pair-first", ["s", "s", "t"], true],
|
const cache = memory(document);
|
||||||
["pair-last", ["t", "s", "s"], true],
|
await cache.put(document, "h1", { a: "s1", b: "t1" });
|
||||||
["pair-split", ["s", "t", "s"], true],
|
await cache.put(document, "h2", { a: "s2", b: "t2" });
|
||||||
["distinct", ["r", "s", "t"], false],
|
await assert.rejects(cache.put(document, "p", { a: "s1", b: "t2" }), { message: refusal("h2", "p", "tests.pair", "\u{10400}") });
|
||||||
] as const) {
|
assert.equal(cache.get(document, "p"), undefined);
|
||||||
const file = join(dir, `${name}.cc`);
|
});
|
||||||
const writer = CultCache.builder().withDocumentType(roles).withGenericStore(new SingleFileMessagePackBackingStore(file)).build();
|
|
||||||
for (const [i, a] of values.entries()) await writer.put(roles, `r${i}`, { a });
|
test("a put names its key pair by UTF-16 code unit, not by normalised form", async () => {
|
||||||
const unique = pair({ indexes: { a: "a" } });
|
// "e" U+0302 is the NFD form of U+00EA: equal when normalised, and less by code unit.
|
||||||
const reader = CultCache.builder().withDocumentType(unique).withGenericStore(new SingleFileMessagePackBackingStore(file)).build();
|
const document = item("item");
|
||||||
if (refused) await assert.rejects(reader.pullAllBackingStores(), /both hold unique index 'a'/u, name);
|
const cache = memory(document);
|
||||||
else await reader.pullAllBackingStores();
|
await cache.put(document, "ê", { sku: "v" });
|
||||||
assert.equal(reader.snapshot().length, refused ? 0 : 3, name);
|
await assert.rejects(cache.put(document, "ê", { sku: "v" }), { message: refusal("ê", "ê", "tests.item", "sku") });
|
||||||
}
|
assert.equal(cache.get(document, "ê"), undefined);
|
||||||
} finally {
|
});
|
||||||
await rm(dir, { recursive: true, force: true });
|
|
||||||
}
|
test("a put colliding on two aliases orders them by UTF-16 code unit, not by normalised form", async () => {
|
||||||
|
// U+00EA normalises to e U+0302, which is less than e U+0303 when normalised and greater by code unit.
|
||||||
|
const document = pair({ indexes: { "ê": "a", "ẽ": "b" } });
|
||||||
|
const cache = memory(document);
|
||||||
|
await cache.put(document, "h1", { a: "s1", b: "t1" });
|
||||||
|
await cache.put(document, "h2", { a: "s2", b: "t2" });
|
||||||
|
await assert.rejects(cache.put(document, "p", { a: "s1", b: "t2" }), { message: refusal("h2", "p", "tests.pair", "ẽ") });
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
<PackageId>GameCult.Math</PackageId>
|
<PackageId>GameCult.Math</PackageId>
|
||||||
<!-- MIT overall; math.Phacelle.cs and CultPhasor.cs (and the two Phacelle HLSL files) are MPL-2.0; math.Cycles.cs and CultMath.Cycles.hlsl are Apache-2.0. -->
|
<!-- MIT overall; math.Phacelle.cs and CultPhasor.cs (and the two Phacelle HLSL files) are MPL-2.0; math.Cycles.cs and CultMath.Cycles.hlsl are Apache-2.0. -->
|
||||||
<PackageLicenseExpression>MIT AND MPL-2.0 AND Apache-2.0</PackageLicenseExpression>
|
<PackageLicenseExpression>MIT AND MPL-2.0 AND Apache-2.0</PackageLicenseExpression>
|
||||||
<Version>0.5.0</Version>
|
<Version>0.5.1</Version>
|
||||||
<Description>Clean-room C# math shaped for HLSL-style renderer code.</Description>
|
<Description>Clean-room C# math shaped for HLSL-style renderer code.</Description>
|
||||||
<Authors>GameCult</Authors>
|
<Authors>GameCult</Authors>
|
||||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ All notable changes to this package are documented in this file.
|
||||||
Earlier releases (0.2.0-0.2.3) predate this package's changelog and are not
|
Earlier releases (0.2.0-0.2.3) predate this package's changelog and are not
|
||||||
backfilled here; see `docs/semver-policy.md` for why.
|
backfilled here; see `docs/semver-policy.md` for why.
|
||||||
|
|
||||||
## [Unreleased]
|
## [0.5.1] - 2026-11-11
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
|
|
||||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,7 +1,7 @@
|
||||||
{
|
{
|
||||||
"name": "org.gamecult.cultmath",
|
"name": "org.gamecult.cultmath",
|
||||||
"displayName": "CultMath",
|
"displayName": "CultMath",
|
||||||
"version": "0.5.0",
|
"version": "0.5.1",
|
||||||
"unity": "2021.3",
|
"unity": "2021.3",
|
||||||
"description": "Precompiled CultMath numeric primitives and matching HLSL semantics for GameCult Unity runtimes.",
|
"description": "Precompiled CultMath numeric primitives and matching HLSL semantics for GameCult Unity runtimes.",
|
||||||
"license": "MIT AND MPL-2.0 AND Apache-2.0",
|
"license": "MIT AND MPL-2.0 AND Apache-2.0",
|
||||||
|
|
|
||||||
|
|
@ -567,8 +567,14 @@ test("a size the runtime cannot allocate is too-large in both exports, not a Ran
|
||||||
const count = 2 ** 15;
|
const count = 2 ** 15;
|
||||||
const huge = { ...manifest, sizeBytes: count * int32Max, chunks: Array.from({ length: count }, (_, index) => ({ ...manifest.chunks[0]!, offset: index * int32Max, sizeBytes: int32Max })) };
|
const huge = { ...manifest, sizeBytes: count * int32Max, chunks: Array.from({ length: count }, (_, index) => ({ ...manifest.chunks[0]!, offset: index * int32Max, sizeBytes: int32Max })) };
|
||||||
const server = double(() => { throw new Error("no request expected"); });
|
const server = double(() => { throw new Error("no request expected"); });
|
||||||
await assert.rejects(fetchVerifiedContent(ENDPOINT, huge, { fetch: server.fetch, maxBytes: size }), codeOf("too-large"));
|
// a host with overcommit hands out 64 TiB lazily, so the refusal is made certain: the runtime cannot allocate it
|
||||||
await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, size, { fetch: server.fetch, maxBytes: size }), codeOf("too-large"));
|
const native = globalThis.Uint8Array;
|
||||||
|
const refusing = new Proxy(native, { construct: (target, args, receiver) => { if (typeof args[0] === "number" && args[0] > 2 ** 40) throw new RangeError("Array buffer allocation failed"); return Reflect.construct(target, args, receiver); } });
|
||||||
|
globalThis.Uint8Array = refusing;
|
||||||
|
try {
|
||||||
|
await assert.rejects(fetchVerifiedContent(ENDPOINT, huge, { fetch: server.fetch, maxBytes: size }), codeOf("too-large"));
|
||||||
|
await assert.rejects(fetchVerifiedChunk(ENDPOINT, manifest.chunks[0]!.chunkHash!, size, { fetch: server.fetch, maxBytes: size }), codeOf("too-large"));
|
||||||
|
} finally { globalThis.Uint8Array = native; }
|
||||||
assert.equal(server.calls.length, 0);
|
assert.equal(server.calls.length, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -43,9 +43,10 @@ from cultmesh_py import (
|
||||||
|
|
||||||
note_doc = define_database_entry_type(
|
note_doc = define_database_entry_type(
|
||||||
"mesh.note",
|
"mesh.note",
|
||||||
[("body", 0), ("name", 1), ("kind", 2)],
|
[("body", 0), ("name", 1), ("kind", 2), ("slug", 3)],
|
||||||
name="name",
|
name="name",
|
||||||
indexes={"kind": "kind"},
|
indexes={"slug": "slug"}, # unique: no two notes hold one slug; watch_by_index and get_key_by_index look it up
|
||||||
|
roles={"kind": "kind"}, # many notes share a kind: a selection alias, never a lookup
|
||||||
)
|
)
|
||||||
node = CultMesh.start_node(
|
node = CultMesh.start_node(
|
||||||
"mesh.cc",
|
"mesh.cc",
|
||||||
|
|
@ -55,13 +56,13 @@ node = CultMesh.start_node(
|
||||||
node.database.register_document(note_doc)
|
node.database.register_document(note_doc)
|
||||||
unsubscribe = node.database.watch_record(note_doc, "note:1", lambda change: print(change.change_kind))
|
unsubscribe = node.database.watch_record(note_doc, "note:1", lambda change: print(change.change_kind))
|
||||||
unsubscribe_named = node.database.watch_by_name(note_doc, "intro", lambda change: print(change.record_key))
|
unsubscribe_named = node.database.watch_by_name(note_doc, "intro", lambda change: print(change.record_key))
|
||||||
unsubscribe_kind = node.database.watch_by_index(note_doc, "kind", "guide", lambda change: print(change.record_key))
|
unsubscribe_slug = node.database.watch_by_index(note_doc, "slug", "intro-guide", lambda change: print(change.record_key))
|
||||||
node.database.put(note_doc, "note:1", {"body": "hello", "name": "intro", "kind": "guide"})
|
node.database.put(note_doc, "note:1", {"body": "hello", "name": "intro", "kind": "guide", "slug": "intro-guide"})
|
||||||
live_note = node.database.get_required(note_doc, "note:1")
|
live_note = node.database.get_required(note_doc, "note:1")
|
||||||
put_message = node.database.put_raw_message(
|
put_message = node.database.put_raw_message(
|
||||||
note_doc,
|
note_doc,
|
||||||
"note:2",
|
"note:2",
|
||||||
{"body": "wire me", "name": "wire", "kind": "interop"},
|
{"body": "wire me", "name": "wire", "kind": "interop", "slug": "wire-me"},
|
||||||
shard_id="interop",
|
shard_id="interop",
|
||||||
shard_epoch=1,
|
shard_epoch=1,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ from pathlib import Path
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
|
|
||||||
from cultcache_py import CultCache, CultCacheEnvelope, SingleFileMessagePackBackingStore
|
from cultcache_py import CultCache, CultCacheEnvelope, SingleFileMessagePackBackingStore
|
||||||
from cultcache_py.cache import CultCacheError
|
from cultcache_py.cache import CultCacheError, index_text
|
||||||
from cultcache_py.documents import DocumentDefinition, extract_value
|
from cultcache_py.documents import DocumentDefinition, extract_value
|
||||||
from cultnet_py import (
|
from cultnet_py import (
|
||||||
CultNetAppliedRecord,
|
CultNetAppliedRecord,
|
||||||
|
|
@ -520,7 +520,7 @@ class CultMeshDatabase:
|
||||||
raise ValueError(f"Document type {document.type!r} has no index {index!r}")
|
raise ValueError(f"Document type {document.type!r} has no index {index!r}")
|
||||||
|
|
||||||
def filtered(change: CultMeshDatabaseChange) -> None:
|
def filtered(change: CultMeshDatabaseChange) -> None:
|
||||||
if self._change_matches_extractor(change, extractor, value):
|
if self._change_matches_extractor(change, extractor, value, index_text):
|
||||||
callback(self._change_as_document(change, requested))
|
callback(self._change_as_document(change, requested))
|
||||||
|
|
||||||
return self.watch(filtered, document=document)
|
return self.watch(filtered, document=document)
|
||||||
|
|
@ -1295,10 +1295,11 @@ class CultMeshDatabase:
|
||||||
change: CultMeshDatabaseChange,
|
change: CultMeshDatabaseChange,
|
||||||
extractor: str | Callable[[Any], str | int | float | bool | None],
|
extractor: str | Callable[[Any], str | int | float | bool | None],
|
||||||
expected: str,
|
expected: str,
|
||||||
|
render: Callable[[Any], str | None] = str,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
return (
|
return (
|
||||||
self._value_matches_extractor(change.value, extractor, expected)
|
self._value_matches_extractor(change.value, extractor, expected, render)
|
||||||
or self._value_matches_extractor(change.previous_value, extractor, expected)
|
or self._value_matches_extractor(change.previous_value, extractor, expected, render)
|
||||||
)
|
)
|
||||||
|
|
||||||
def _value_matches_extractor(
|
def _value_matches_extractor(
|
||||||
|
|
@ -1306,11 +1307,12 @@ class CultMeshDatabase:
|
||||||
value: Any | None,
|
value: Any | None,
|
||||||
extractor: str | Callable[[Any], str | int | float | bool | None],
|
extractor: str | Callable[[Any], str | int | float | bool | None],
|
||||||
expected: str,
|
expected: str,
|
||||||
|
render: Callable[[Any], str | None] = str,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
if value is None:
|
if value is None:
|
||||||
return False
|
return False
|
||||||
actual = extract_value(value, extractor)
|
actual = extract_value(value, extractor)
|
||||||
return actual is not None and str(actual) == expected
|
return actual is not None and render(actual) == expected
|
||||||
|
|
||||||
def _previous_values_for_snapshot_response(self, response: dict[str, Any]) -> dict[tuple[str, str], Any]:
|
def _previous_values_for_snapshot_response(self, response: dict[str, Any]) -> dict[tuple[str, str], Any]:
|
||||||
if response.get("schemaVersion") != "cultnet.snapshot_response_raw.v0":
|
if response.get("schemaVersion") != "cultnet.snapshot_response_raw.v0":
|
||||||
|
|
|
||||||
84
packages/cultmesh-py/tests/test_unique_index_watch.py
Normal file
84
packages/cultmesh-py/tests/test_unique_index_watch.py
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from cultcache_py import define_database_entry_type
|
||||||
|
from cultcache_py.cache import CultCacheError
|
||||||
|
from cultmesh_py import CultMesh
|
||||||
|
|
||||||
|
# variants:ruling:unique-indexes. `indexes` are unique lookups, `roles` are selection aliases. A watch by index reads the
|
||||||
|
# lookup's own canonical text (cultcache_py.cache.index_text), never a second rendering of the value.
|
||||||
|
|
||||||
|
|
||||||
|
class WatchByIndexTests(unittest.TestCase):
|
||||||
|
def test_the_watch_and_the_lookup_agree_on_the_text_of_a_value(self) -> None:
|
||||||
|
document = define_database_entry_type("mesh.priced", [("price", 0)], indexes={"price": "price"})
|
||||||
|
node = CultMesh.create_node(runtime_id="mesh-watch-text")
|
||||||
|
node.database.register_document(document)
|
||||||
|
cases = [
|
||||||
|
(2.0, "2", "2.0"),
|
||||||
|
(0.1, "0.1000000000000000055511151231257827021181583404541015625", "0.1"),
|
||||||
|
(-0.0, "0", "-0.0"),
|
||||||
|
]
|
||||||
|
for index, (value, text, str_text) in enumerate(cases):
|
||||||
|
with self.subTest(value=value):
|
||||||
|
key = f"item:{index}"
|
||||||
|
heard: list[str] = []
|
||||||
|
misheard: list[str] = []
|
||||||
|
node.database.watch_by_index(document, "price", text, lambda change: heard.append(change.record_key))
|
||||||
|
node.database.watch_by_index(document, "price", str_text, lambda change: misheard.append(change.record_key))
|
||||||
|
node.database.put(document, key, {"price": value})
|
||||||
|
self.assertEqual(node.database.get_by_index(document, "price", text), {"price": value})
|
||||||
|
self.assertEqual(heard, [key])
|
||||||
|
if str_text != text:
|
||||||
|
self.assertEqual(misheard, [])
|
||||||
|
self.assertIsNone(node.database.get_by_index(document, "price", str_text))
|
||||||
|
|
||||||
|
def test_a_watch_hears_a_record_leaving_a_value_by_the_canonical_text(self) -> None:
|
||||||
|
document = define_database_entry_type("mesh.leaving", [("price", 0)], indexes={"price": "price"})
|
||||||
|
node = CultMesh.create_node(runtime_id="mesh-watch-leaving")
|
||||||
|
node.database.register_document(document)
|
||||||
|
node.database.put(document, "item:1", {"price": 2.0})
|
||||||
|
heard: list[str] = []
|
||||||
|
node.database.watch_by_index(document, "price", "2", lambda change: heard.append(change.record_key))
|
||||||
|
node.database.put(document, "item:1", {"price": 3.0})
|
||||||
|
self.assertEqual(heard, ["item:1"])
|
||||||
|
|
||||||
|
def test_a_blank_value_is_never_indexed_so_never_watched(self) -> None:
|
||||||
|
document = define_database_entry_type("mesh.blank", [("sku", 0)], indexes={"sku": "sku"})
|
||||||
|
node = CultMesh.create_node(runtime_id="mesh-watch-blank")
|
||||||
|
node.database.register_document(document)
|
||||||
|
heard: list[str] = []
|
||||||
|
node.database.watch_by_index(document, "sku", " ", lambda change: heard.append(change.record_key))
|
||||||
|
node.database.put(document, "item:1", {"sku": " "})
|
||||||
|
self.assertEqual(heard, [])
|
||||||
|
|
||||||
|
def test_a_role_alias_is_not_watchable_by_index(self) -> None:
|
||||||
|
document = define_database_entry_type("mesh.roles", [("kind", 0)], roles={"kind": "kind"})
|
||||||
|
node = CultMesh.create_node(runtime_id="mesh-watch-role")
|
||||||
|
node.database.register_document(document)
|
||||||
|
with self.assertRaisesRegex(ValueError, "has no index"):
|
||||||
|
node.database.watch_by_index(document, "kind", "guide", lambda _: None)
|
||||||
|
|
||||||
|
|
||||||
|
class ReadmeTests(unittest.TestCase):
|
||||||
|
def test_the_readme_note_document_admits_many_notes_of_one_kind(self) -> None:
|
||||||
|
readme = (Path(__file__).parents[1] / "README.md").read_text(encoding="utf-8")
|
||||||
|
declaration = re.search(r"^note_doc = define_database_entry_type\(\n.*?^\)\n", readme, re.S | re.M)
|
||||||
|
self.assertIsNotNone(declaration)
|
||||||
|
scope: dict = {"define_database_entry_type": define_database_entry_type}
|
||||||
|
exec(declaration.group(0), scope) # noqa: S102 - the README's own declaration
|
||||||
|
note_doc = scope["note_doc"]
|
||||||
|
node = CultMesh.create_node(runtime_id="mesh-readme")
|
||||||
|
node.database.register_document(note_doc)
|
||||||
|
node.database.put(note_doc, "note:1", {"body": "a", "name": "one", "kind": "guide", "slug": "one"})
|
||||||
|
node.database.put(note_doc, "note:2", {"body": "b", "name": "two", "kind": "guide", "slug": "two"})
|
||||||
|
self.assertEqual(node.database.get_by_index(note_doc, "slug", "two")["body"], "b")
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
node.database.put(note_doc, "note:3", {"body": "c", "name": "three", "kind": "guide", "slug": "one"})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -2,6 +2,8 @@
|
||||||
|
|
||||||
## Unreleased (breaking; version 0.x, so the next minor)
|
## Unreleased (breaking; version 0.x, so the next minor)
|
||||||
|
|
||||||
|
- `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out.
|
||||||
|
- `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move.
|
||||||
- Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client.
|
- Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client.
|
||||||
- Added `message_invalid` to `CULTNET_ERROR_CODES` and `message_invalid_error(field, reason)`, byte-identical to the C# `ForMessageInvalid`. `cultmesh-py` refuses a malformed put or delete with it instead of the out-of-schema `malformed_document_put` and `malformed_document_delete`, which are removed.
|
- Added `message_invalid` to `CULTNET_ERROR_CODES` and `message_invalid_error(field, reason)`, byte-identical to the C# `ForMessageInvalid`. `cultmesh-py` refuses a malformed put or delete with it instead of the out-of-schema `malformed_document_put` and `malformed_document_delete`, which are removed.
|
||||||
- Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per
|
- Added `CultNetRudpSocketTransportServer` (`cultnet_py.rudp_server`): one UDP socket serving many RUDP peers, one session per
|
||||||
|
|
|
||||||
|
|
@ -692,7 +692,7 @@ simulation_fact_document: DocumentDefinition[CultMeshSimulationFact] = define_da
|
||||||
],
|
],
|
||||||
cls=CultMeshSimulationFact,
|
cls=CultMeshSimulationFact,
|
||||||
name="fact_id",
|
name="fact_id",
|
||||||
indexes={"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"},
|
roles={"shard": "shard_id", "frame": "frame", "subject": "subject_id", "claim_kind": "claim_kind"},
|
||||||
schema_name=SIMULATION_FACT_DOCUMENT_TYPE,
|
schema_name=SIMULATION_FACT_DOCUMENT_TYPE,
|
||||||
schema_version=SIMULATION_FACT_SCHEMA_VERSION,
|
schema_version=SIMULATION_FACT_SCHEMA_VERSION,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -429,9 +429,11 @@ def wire_message_schema_properties(schema_version: str) -> dict[str, Any]:
|
||||||
"messageId": common["messageId"],
|
"messageId": common["messageId"],
|
||||||
"subscriptionId": common["subscriptionId"],
|
"subscriptionId": common["subscriptionId"],
|
||||||
"changeKind": {"enum": ["added", "updated", "removed"]},
|
"changeKind": {"enum": ["added", "updated", "removed"]},
|
||||||
"document": {"$ref": "#/$defs/rawDocumentRecord"},
|
"document": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/rawDocumentRecord"}]},
|
||||||
"schemaId": common["schemaId"],
|
"schemaId": {"type": ["string", "null"], "minLength": 1},
|
||||||
"recordKey": common["recordKey"],
|
"recordKey": {"type": ["string", "null"], "minLength": 1},
|
||||||
|
"commitId": {"type": ["string", "null"], "minLength": 1},
|
||||||
|
"commitCount": {"type": ["integer", "null"], "minimum": 2},
|
||||||
},
|
},
|
||||||
"cultnet.shard_catalog_request.v0": {
|
"cultnet.shard_catalog_request.v0": {
|
||||||
**common,
|
**common,
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,10 @@ class CultNetDatabaseChange:
|
||||||
raw_document: CultNetRawDocumentRecord | None = None
|
raw_document: CultNetRawDocumentRecord | None = None
|
||||||
schema_id: str | None = None
|
schema_id: str | None = None
|
||||||
record_key: str | None = None
|
record_key: str | None = None
|
||||||
|
commit_id: str | None = None
|
||||||
|
"""Set on every change of one multi-record commit, the same for all of them; None outside one (C# writes nil)."""
|
||||||
|
commit_count: int | None = None
|
||||||
|
"""How many changes the commit holds (>= 2); None outside one. Carried data: Python applies no batch."""
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_wire(cls, message: dict[str, Any]) -> "CultNetDatabaseChange":
|
def from_wire(cls, message: dict[str, Any]) -> "CultNetDatabaseChange":
|
||||||
|
|
@ -31,6 +35,12 @@ class CultNetDatabaseChange:
|
||||||
if raw_document is not None:
|
if raw_document is not None:
|
||||||
schema_id = schema_id or raw_document.schema_id
|
schema_id = schema_id or raw_document.schema_id
|
||||||
record_key = record_key or raw_document.record_key
|
record_key = record_key or raw_document.record_key
|
||||||
|
commit_id = message.get("commitId")
|
||||||
|
if commit_id is not None and not (isinstance(commit_id, str) and commit_id):
|
||||||
|
raise ValueError(f"database change commitId must be a non-empty string, received {commit_id!r}")
|
||||||
|
commit_count = message.get("commitCount")
|
||||||
|
if commit_count is not None and (isinstance(commit_count, bool) or not isinstance(commit_count, int) or commit_count < 2):
|
||||||
|
raise ValueError(f"database change commitCount must be an integer of at least 2, received {commit_count!r}")
|
||||||
return cls(
|
return cls(
|
||||||
message_id=str(message.get("messageId") or ""),
|
message_id=str(message.get("messageId") or ""),
|
||||||
subscription_id=str(message.get("subscriptionId") or ""),
|
subscription_id=str(message.get("subscriptionId") or ""),
|
||||||
|
|
@ -39,6 +49,8 @@ class CultNetDatabaseChange:
|
||||||
raw_document=raw_document,
|
raw_document=raw_document,
|
||||||
schema_id=schema_id,
|
schema_id=schema_id,
|
||||||
record_key=record_key,
|
record_key=record_key,
|
||||||
|
commit_id=commit_id,
|
||||||
|
commit_count=commit_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
def to_wire(self) -> dict[str, Any]:
|
def to_wire(self) -> dict[str, Any]:
|
||||||
|
|
@ -56,6 +68,10 @@ class CultNetDatabaseChange:
|
||||||
wire["schemaId"] = self.schema_id
|
wire["schemaId"] = self.schema_id
|
||||||
if self.record_key is not None:
|
if self.record_key is not None:
|
||||||
wire["recordKey"] = self.record_key
|
wire["recordKey"] = self.record_key
|
||||||
|
if self.commit_id is not None:
|
||||||
|
wire["commitId"] = self.commit_id
|
||||||
|
if self.commit_count is not None:
|
||||||
|
wire["commitCount"] = self.commit_count
|
||||||
return wire
|
return wire
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,8 @@ from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable
|
from typing import Callable
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
from cultcache_py import CultCache, define_database_entry_type
|
import msgpack
|
||||||
|
from cultcache_py import CultCache, SchemaConflictError, define_database_entry_type
|
||||||
from cultnet_py.benchmark import run_benchmark
|
from cultnet_py.benchmark import run_benchmark
|
||||||
from cultnet_py.compare_csharp import (
|
from cultnet_py.compare_csharp import (
|
||||||
DEFAULT_PARITY_THRESHOLD,
|
DEFAULT_PARITY_THRESHOLD,
|
||||||
|
|
@ -1566,6 +1567,51 @@ class CultNetTests(unittest.TestCase):
|
||||||
self.assertEqual(delete_change.schema_id, "schema-note")
|
self.assertEqual(delete_change.schema_id, "schema-note")
|
||||||
self.assertEqual(delete_change.to_wire()["recordKey"], "note:1")
|
self.assertEqual(delete_change.to_wire()["recordKey"], "note:1")
|
||||||
|
|
||||||
|
def test_cultnet_database_change_commit_fields(self) -> None:
|
||||||
|
base = {
|
||||||
|
"schemaVersion": "cultnet.database_change_raw.v0",
|
||||||
|
"messageId": "change-commit",
|
||||||
|
"subscriptionId": "sub-1",
|
||||||
|
"changeKind": "removed",
|
||||||
|
"schemaId": "schema-note",
|
||||||
|
"recordKey": "note:1",
|
||||||
|
}
|
||||||
|
grouped = CultNetDatabaseChange.from_wire({**base, "commitId": "commit-7", "commitCount": 3})
|
||||||
|
self.assertEqual((grouped.commit_id, grouped.commit_count), ("commit-7", 3))
|
||||||
|
self.assertEqual(grouped.to_wire()["commitId"], "commit-7")
|
||||||
|
self.assertEqual(grouped.to_wire()["commitCount"], 3)
|
||||||
|
self.assertEqual(CultNetDatabaseChange.from_wire(grouped.to_wire()), grouped)
|
||||||
|
|
||||||
|
for absent in (base, {**base, "commitId": None, "commitCount": None}): # C# writes nil outside a commit
|
||||||
|
change = CultNetDatabaseChange.from_wire(absent)
|
||||||
|
self.assertIsNone(change.commit_id)
|
||||||
|
self.assertIsNone(change.commit_count)
|
||||||
|
self.assertNotIn("commitId", change.to_wire())
|
||||||
|
self.assertNotIn("commitCount", change.to_wire())
|
||||||
|
|
||||||
|
for field, bad in (
|
||||||
|
("commitId", ""),
|
||||||
|
("commitId", 7),
|
||||||
|
("commitCount", 1),
|
||||||
|
("commitCount", 0),
|
||||||
|
("commitCount", True),
|
||||||
|
("commitCount", 2.0),
|
||||||
|
("commitCount", "3"),
|
||||||
|
):
|
||||||
|
with self.subTest(field=field, bad=bad), self.assertRaisesRegex(ValueError, field):
|
||||||
|
CultNetDatabaseChange.from_wire({**base, field: bad})
|
||||||
|
|
||||||
|
def test_a_two_change_commit_written_by_csharp_parses(self) -> None:
|
||||||
|
# tests/vectors/database-change-commit/two-change-commit.msgpack: the two change messages of one commit, written by the
|
||||||
|
# C# serializer (CultNetReplicaCommitTests), so a rename or retyping of commitId/commitCount fails here.
|
||||||
|
vector = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "database-change-commit" / "two-change-commit.msgpack"
|
||||||
|
messages = list(msgpack.Unpacker(io.BytesIO(vector.read_bytes()), raw=False))
|
||||||
|
self.assertEqual(len(messages), 2)
|
||||||
|
changes = [CultNetDatabaseChange.from_wire(message) for message in messages]
|
||||||
|
self.assertEqual([change.commit_count for change in changes], [2, 2])
|
||||||
|
self.assertEqual({change.commit_id for change in changes}, {"9f1c0f0e4b7a4c1f9d2e"})
|
||||||
|
self.assertEqual([change.to_wire()["commitCount"] for change in changes], [2, 2])
|
||||||
|
|
||||||
def test_cultnet_raw_put_helper_carries_message_id(self) -> None:
|
def test_cultnet_raw_put_helper_carries_message_id(self) -> None:
|
||||||
put = document_put_raw(
|
put = document_put_raw(
|
||||||
message_id="put-1",
|
message_id="put-1",
|
||||||
|
|
@ -1896,6 +1942,35 @@ class CultNetTests(unittest.TestCase):
|
||||||
self.assertEqual(raised.exception.response["messageId"], "subscription-error")
|
self.assertEqual(raised.exception.response["messageId"], "subscription-error")
|
||||||
thread.join(2.0)
|
thread.join(2.0)
|
||||||
|
|
||||||
|
def test_replication_refuses_a_remote_record_of_another_type_at_a_held_key_and_changes_nothing(self) -> None:
|
||||||
|
# variants:ruling:cache-refuses-retype: a cache holds one record per key whatever its type, so a peer that reuses a
|
||||||
|
# key across types is refused where the record is applied; replication does not catch it.
|
||||||
|
first = define_database_entry_type("replica.first", [("name", 0), ("category", 1), ("value", 2, 0)], cls=Item)
|
||||||
|
second = define_database_entry_type("replica.second", [("name", 0), ("category", 1), ("value", 2, 0)], cls=Item)
|
||||||
|
cache = CultCache()
|
||||||
|
cache.register_document_type(first)
|
||||||
|
cache.register_document_type(second)
|
||||||
|
cache.put(first, "shared", Item("held", "", 0))
|
||||||
|
|
||||||
|
def raw(document):
|
||||||
|
return {
|
||||||
|
"schemaId": document.catalog_entry().schema_id,
|
||||||
|
"recordKey": "shared",
|
||||||
|
"storedAt": "2026-06-13T00:00:00Z",
|
||||||
|
"payloadEncoding": "messagepack",
|
||||||
|
"payload": document.encode_payload(Item("remote", "", 0)),
|
||||||
|
}
|
||||||
|
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
apply_raw_document_record(cache, schema_document_map([first, second]), raw(second))
|
||||||
|
snapshot = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": "s", "documents": [raw(second)]}
|
||||||
|
with self.assertRaises(SchemaConflictError):
|
||||||
|
apply_raw_snapshot(cache, [first, second], snapshot)
|
||||||
|
self.assertEqual(cache.get_required(first, "shared").name, "held")
|
||||||
|
self.assertIsNone(cache.get(second, "shared"))
|
||||||
|
apply_raw_document_record(cache, schema_document_map([first, second]), raw(first)) # the same type still replaces
|
||||||
|
self.assertEqual(cache.get_required(first, "shared").name, "remote")
|
||||||
|
|
||||||
def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None:
|
def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None:
|
||||||
document = define_database_entry_type(
|
document = define_database_entry_type(
|
||||||
"replica.item",
|
"replica.item",
|
||||||
|
|
@ -2269,9 +2344,12 @@ class CultNetTests(unittest.TestCase):
|
||||||
|
|
||||||
change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"])
|
change_schema = json.loads(by_version["cultnet.database_change_raw.v0"]["schemaJson"])
|
||||||
self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"])
|
self.assertEqual(change_schema["properties"]["changeKind"]["enum"], ["added", "updated", "removed"])
|
||||||
self.assertIn("document", change_schema["properties"])
|
# The C# writer sets every key, nil where unset (a removed change's document, an added change's recordKey and schemaId).
|
||||||
self.assertIn("schemaId", change_schema["properties"])
|
self.assertEqual(change_schema["properties"]["document"]["oneOf"][0], {"type": "null"})
|
||||||
self.assertIn("recordKey", change_schema["properties"])
|
self.assertEqual(change_schema["properties"]["schemaId"], {"type": ["string", "null"], "minLength": 1})
|
||||||
|
self.assertEqual(change_schema["properties"]["recordKey"], {"type": ["string", "null"], "minLength": 1})
|
||||||
|
self.assertEqual(change_schema["properties"]["commitId"], {"type": ["string", "null"], "minLength": 1})
|
||||||
|
self.assertEqual(change_schema["properties"]["commitCount"], {"type": ["integer", "null"], "minimum": 2})
|
||||||
|
|
||||||
log_schema = json.loads(by_version["cultnet.shard_log_response.v0"]["schemaJson"])
|
log_schema = json.loads(by_version["cultnet.shard_log_response.v0"]["schemaJson"])
|
||||||
self.assertIn("entries", log_schema["required"])
|
self.assertIn("entries", log_schema["required"])
|
||||||
|
|
|
||||||
43
packages/cultnet-rs/tests/database_change_commit_vector.rs
Normal file
43
packages/cultnet-rs/tests/database_change_commit_vector.rs
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
//! `tests/vectors/database-change-commit/two-change-commit.msgpack` is written only by the C# reference: two
|
||||||
|
//! `cultnet.database_change_raw.v0` messages of one commit, every key present, nil where unset. cultnet-rs has no typed
|
||||||
|
//! `database_change_raw` message yet, so this walks the stream with the same `rmpv` values its wire decoder builds on and
|
||||||
|
//! pins the keys and values a Rust adopter will read. A rename of `commitId` or `commitCount` in the C# writer fails here.
|
||||||
|
|
||||||
|
use rmpv::Value;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
fn messages() -> Vec<Value> {
|
||||||
|
let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../tests/vectors/database-change-commit/two-change-commit.msgpack");
|
||||||
|
let bytes = std::fs::read(path).expect("the vector is committed");
|
||||||
|
let mut cursor = Cursor::new(bytes.as_slice());
|
||||||
|
let mut out = Vec::new();
|
||||||
|
while (cursor.position() as usize) < bytes.len() {
|
||||||
|
out.push(rmpv::decode::read_value(&mut cursor).expect("a msgpack value"));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn field<'a>(message: &'a Value, key: &str) -> &'a Value {
|
||||||
|
message
|
||||||
|
.as_map()
|
||||||
|
.and_then(|entries| entries.iter().find(|(k, _)| k.as_str() == Some(key)))
|
||||||
|
.map(|(_, v)| v)
|
||||||
|
.unwrap_or_else(|| panic!("the C# writer wrote no key {key}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_two_change_commit_written_by_csharp_carries_its_commit_id_count_and_changes() {
|
||||||
|
let messages = messages();
|
||||||
|
assert_eq!(messages.len(), 2);
|
||||||
|
for message in &messages {
|
||||||
|
assert_eq!(field(message, "schemaVersion").as_str(), Some("cultnet.database_change_raw.v0"));
|
||||||
|
assert_eq!(field(message, "commitId").as_str(), Some("9f1c0f0e4b7a4c1f9d2e"));
|
||||||
|
assert_eq!(field(message, "commitCount").as_u64(), Some(2));
|
||||||
|
assert_eq!(field(message, "changeKind").as_str(), Some("removed"));
|
||||||
|
assert_eq!(field(message, "subscriptionId").as_str(), Some("sub-1"));
|
||||||
|
assert_eq!(field(message, "schemaId").as_str(), Some("schema-note"));
|
||||||
|
assert!(field(message, "document").is_nil(), "a removed change's document is nil");
|
||||||
|
}
|
||||||
|
let keys: Vec<_> = messages.iter().map(|m| field(m, "recordKey").as_str().unwrap().to_string()).collect();
|
||||||
|
assert_eq!(keys, ["note:a", "note:b"]);
|
||||||
|
}
|
||||||
|
|
@ -5,8 +5,15 @@ All notable changes to `@gamecult/cultnet-ts` are recorded here, following
|
||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
### Breaking
|
||||||
|
|
||||||
|
- `decodeManifest` refuses a manifest with bytes after its first MessagePack value (`CDN artifact manifest has N bytes after its first MessagePack value.`), as the C# reference does; before, the tail was ignored. `contracts/cultmesh/content-acceptance-vectors.json` now holds `trailing_nil`, `trailing_reserved_code` and `trailing_truncated_array` as refused.
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- Fork card (`contracts/cultnet/governance/fork-card.md`): `raised_at`, a required key holding an RFC3339 instant string or nil (the question's first admission time), outside `options_hash`; the card vectors are re-pinned with the key, three negatives (epoch integer, date-only string, missing key) and a nil-`raised_at` valid card; every `optionsHash` digest is unchanged.
|
||||||
|
- Fork card reader views (`fork-card.md`, "Reader views"): the card gains `decider_count` (required, 1 to 64) and `deciders` may be empty (ids withheld from a non-decider); the record gains `answerer_count` (distinct answerers over every stored answer, refusals included); `answerer` is nullable inside a record shown to a non-decider. Card and record vectors are re-pinned (card map header `DE0011` to `DE0012`, record `85` to `86`); withheld twins, the R-card, R-record, R-entry, R-list and R-answer-result rule vectors and a no-leak byte check are new; every `optionsHash` and `bodyDigests` digest is unchanged.
|
||||||
|
- Fork card vectors: a nil `answerer` is now pinned for Revision and Retraction as well as Counted and Refused (`deskAnswer` `withheld_answerer_revision`, `withheld_answerer_retraction`; entries `open_counted_revision_and_retraction` and its withheld twin; a list scenario pair; negatives for a nil answerer in an `answer_result` and for a named Retraction or an over-count in a withheld entry). Additive: every existing vector, `optionsHash` and `bodyDigests` digest is unchanged.
|
||||||
- `WriteTracker`: a write sent after `sessionEnd` settles unconfirmed `session_ended` when it is sent (it stayed unanswered forever). The replay runner now fails on an event key outside `scenarios.semantics` and on a vector count that differs from the file's `counts`; seven tracker vectors are new. A blank text in a raw frame is Unicode White_Space, not `String.prototype.trim()`.
|
- `WriteTracker`: a write sent after `sessionEnd` settles unconfirmed `session_ended` when it is sent (it stayed unanswered forever). The replay runner now fails on an event key outside `scenarios.semantics` and on a vector count that differs from the file's `counts`; seven tracker vectors are new. A blank text in a raw frame is Unicode White_Space, not `String.prototype.trim()`.
|
||||||
- Write receipt, client side (`contracts/cultnet/interop/write-receipt.md`): `cultnet.write_admitted.v0` and `cultnet.error.v1` (v0 plus `messageId`; its closed code set adds `limit_reached`, `CULTNET_ERROR_V1_CODES`) parse and encode, and `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS` / `helloAdvertisesWriteReceipts` name the hello capability. `cultnet.error.v0` and its code set are unchanged. `WriteTracker` settles each write as admitted, refused or unconfirmed (windows `RECEIPT_RESULT_WINDOW_MS` 5000 and `LEGACY_RESULT_WINDOW_MS` 500, counted from the send; late answers recorded as `admitted_late` / `refused_late`), and `tools/replay-write-receipt-vectors.mjs` replays the vectors against it (see README, "Write receipt replay runner"). The interop schemas are read from `contracts/cultnet/interop/`.
|
- Write receipt, client side (`contracts/cultnet/interop/write-receipt.md`): `cultnet.write_admitted.v0` and `cultnet.error.v1` (v0 plus `messageId`; its closed code set adds `limit_reached`, `CULTNET_ERROR_V1_CODES`) parse and encode, and `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS` / `helloAdvertisesWriteReceipts` name the hello capability. `cultnet.error.v0` and its code set are unchanged. `WriteTracker` settles each write as admitted, refused or unconfirmed (windows `RECEIPT_RESULT_WINDOW_MS` 5000 and `LEGACY_RESULT_WINDOW_MS` 500, counted from the send; late answers recorded as `admitted_late` / `refused_late`), and `tools/replay-write-receipt-vectors.mjs` replays the vectors against it (see README, "Write receipt replay runner"). The interop schemas are read from `contracts/cultnet/interop/`.
|
||||||
- `message_invalid` joins the closed `cultnet.error.v0` code set, with `cultNetMessageInvalidError(field, reason)` (`details.field` names the request field at fault), byte-identical to the C# `ForMessageInvalid`.
|
- `message_invalid` joins the closed `cultnet.error.v0` code set, with `cultNetMessageInvalidError(field, reason)` (`details.field` names the request field at fault), byte-identical to the C# `ForMessageInvalid`.
|
||||||
|
|
|
||||||
|
|
@ -144,8 +144,8 @@ imports no `node:` module, so a browser bundle can take it.
|
||||||
past the shared depth limit are refused.
|
past the shared depth limit are refused.
|
||||||
- The decoder reads what MessagePack-CSharp reads and refuses what it refuses:
|
- The decoder reads what MessagePack-CSharp reads and refuses what it refuses:
|
||||||
nil in a string, tag or metadata slot is `null` and is written back as nil,
|
nil in a string, tag or metadata slot is `null` and is written back as nil,
|
||||||
a missing trailing slot takes the reference's default, extra slots and trailing
|
a missing trailing slot takes the reference's default, extra slots are skipped and bytes
|
||||||
bytes are skipped (a chunk reference's fifth slot is its element id, a string
|
after the manifest's first value are refused (a chunk reference's fifth slot is its element id, a string
|
||||||
or nil, `""` when absent; `packContent` writes the decimal offset as the id, as the
|
or nil, `""` when absent; `packContent` writes the decimal offset as the id, as the
|
||||||
reference does), a float-encoded integer is refused, and invalid UTF-8
|
reference does), a float-encoded integer is refused, and invalid UTF-8
|
||||||
decodes to U+FFFD. A nil `chunks` is refused, as the reference's
|
decodes to U+FFFD. A nil `chunks` is refused, as the reference's
|
||||||
|
|
|
||||||
|
|
@ -249,7 +249,7 @@ class Reader {
|
||||||
export function decodeManifest(bytes: Uint8Array): CultMeshCdnArtifactManifest {
|
export function decodeManifest(bytes: Uint8Array): CultMeshCdnArtifactManifest {
|
||||||
checkPayloadDepth(bytes, "CDN artifact manifest");
|
checkPayloadDepth(bytes, "CDN artifact manifest");
|
||||||
const reader = new Reader(bytes);
|
const reader = new Reader(bytes);
|
||||||
// A slot the array does not reach keeps the property initialiser; a slot past the tenth is skipped. Trailing bytes are not read.
|
// A slot the array does not reach keeps the property initialiser; a slot past the tenth is skipped.
|
||||||
const count = reader.array("manifest");
|
const count = reader.array("manifest");
|
||||||
let index = 0;
|
let index = 0;
|
||||||
const slot = <T>(read: () => T, initial: T): T => index++ < count ? read() : initial;
|
const slot = <T>(read: () => T, initial: T): T => index++ < count ? read() : initial;
|
||||||
|
|
@ -267,6 +267,8 @@ export function decodeManifest(bytes: Uint8Array): CultMeshCdnArtifactManifest {
|
||||||
[reader.string("metadata") ?? wrongType("metadata"), reader.string("metadata")] as const), []),
|
[reader.string("metadata") ?? wrongType("metadata"), reader.string("metadata")] as const), []),
|
||||||
};
|
};
|
||||||
for (; index < count; index += 1) reader.skip();
|
for (; index < count; index += 1) reader.skip();
|
||||||
|
// A manifest is one record payload: exactly one MessagePack value, as the reference's one-value rule has it.
|
||||||
|
if (reader.position < bytes.length) throw new Error(`CDN artifact manifest has ${bytes.length - reader.position} bytes after its first MessagePack value.`);
|
||||||
validateManifest(manifest);
|
validateManifest(manifest);
|
||||||
return manifest;
|
return manifest;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
import test from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import Ajv2020 from "ajv/dist/2020.js";
|
||||||
|
import { decodeMulti } from "@msgpack/msgpack";
|
||||||
|
|
||||||
|
import { cultNetSchemas, parseCultNetMessage } from "../src/index";
|
||||||
|
|
||||||
|
// tests/vectors/database-change-commit/two-change-commit.msgpack is written only by the C# reference: two removed
|
||||||
|
// database_change_raw messages of one commit, every key present, nil where unset (document included).
|
||||||
|
const vector = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "database-change-commit", "two-change-commit.msgpack");
|
||||||
|
const changes = () => [...decodeMulti(readFileSync(vector))] as Record<string, unknown>[];
|
||||||
|
|
||||||
|
test("a two-change commit written by C# decodes with its commitId, commitCount and changes", () => {
|
||||||
|
const messages = changes().map((message) => parseCultNetMessage(structuredClone(message)));
|
||||||
|
assert.equal(messages.length, 2);
|
||||||
|
for (const message of messages) {
|
||||||
|
assert.equal(message.schemaVersion, "cultnet.database_change_raw.v0");
|
||||||
|
assert.equal((message as { commitId?: unknown }).commitId, "9f1c0f0e4b7a4c1f9d2e");
|
||||||
|
assert.equal((message as { commitCount?: unknown }).commitCount, 2);
|
||||||
|
assert.equal((message as { changeKind?: unknown }).changeKind, "removed");
|
||||||
|
assert.equal((message as { subscriptionId?: unknown }).subscriptionId, "sub-1");
|
||||||
|
assert.equal((message as { schemaId?: unknown }).schemaId, "schema-note");
|
||||||
|
}
|
||||||
|
assert.deepEqual(messages.map((message) => (message as { recordKey?: unknown }).recordKey), ["note:a", "note:b"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the database_change_raw schema accepts the C# vector's removed changes (document: nil) and still refuses an added change without a document", () => {
|
||||||
|
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
||||||
|
ajv.addSchema(cultNetSchemas.rawDocumentRecordSchema);
|
||||||
|
const validate = ajv.compile(cultNetSchemas.databaseChangeRawSchema);
|
||||||
|
for (const change of changes()) {
|
||||||
|
assert.equal(change.document, null, "the C# writer wrote the key, nil");
|
||||||
|
assert.equal(validate(change), true, JSON.stringify(validate.errors));
|
||||||
|
assert.equal(validate({ ...change, changeKind: "added" }), false, "an added change needs a document");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
@ -90,7 +90,7 @@ const jstr = (s: string): string => {
|
||||||
};
|
};
|
||||||
type Option = { id: string; text: string };
|
type Option = { id: string; text: string };
|
||||||
type Shown = { title: string; context: string; options: Option[]; recommendation: string; depends: string[] };
|
type Shown = { title: string; context: string; options: Option[]; recommendation: string; depends: string[] };
|
||||||
type Card = Shown & { fork_id: string; leans: string | null; deciders: string[]; status: string; ruling: string | null; ruled_from: string[]; options_hash: string };
|
type Card = Shown & { fork_id: string; leans: string | null; raised_at: string | null; deciders: string[]; decider_count: number; status: string; ruling: string | null; ruled_from: string[]; options_hash: string };
|
||||||
// RFC 8785 over the five shown members, every string NFC-normalised first; keys sorted; no numbers in the member set.
|
// RFC 8785 over the five shown members, every string NFC-normalised first; keys sorted; no numbers in the member set.
|
||||||
const nfc = (s: string): string => s.normalize("NFC");
|
const nfc = (s: string): string => s.normalize("NFC");
|
||||||
const canonical = (c: Shown): Buffer =>
|
const canonical = (c: Shown): Buffer =>
|
||||||
|
|
@ -104,6 +104,7 @@ const cardRuleErrors = (c: Card): string[] => {
|
||||||
if (!ids.includes(c.recommendation)) errors.push("recommendation is not an option");
|
if (!ids.includes(c.recommendation)) errors.push("recommendation is not an option");
|
||||||
if (c.leans !== null && !ids.includes(c.leans)) errors.push("leans is not an option");
|
if (c.leans !== null && !ids.includes(c.leans)) errors.push("leans is not an option");
|
||||||
if (new Set(c.deciders).size !== c.deciders.length) errors.push("duplicate deciders");
|
if (new Set(c.deciders).size !== c.deciders.length) errors.push("duplicate deciders");
|
||||||
|
if (c.deciders.length > 0 && c.deciders.length !== c.decider_count) errors.push("R-card: shown deciders are not exactly decider_count ids");
|
||||||
if (c.depends.some((d) => Buffer.byteLength(d, "utf8") > 1000)) errors.push("a depends line is over 1000 UTF-8 bytes");
|
if (c.depends.some((d) => Buffer.byteLength(d, "utf8") > 1000)) errors.push("a depends line is over 1000 UTF-8 bytes");
|
||||||
if (c.status === "open" && c.ruling !== null) errors.push("an open card names a ruling");
|
if (c.status === "open" && c.ruling !== null) errors.push("an open card names a ruling");
|
||||||
if (c.ruling === null && c.ruled_from.length > 0) errors.push("ruled_from without a ruling");
|
if (c.ruling === null && c.ruled_from.length > 0) errors.push("ruled_from without a ruling");
|
||||||
|
|
@ -121,6 +122,15 @@ test("fork card: options hash is SHA-256 of the canonical JSON, computed here, a
|
||||||
const byName = new Map<string, string>(fork.optionsHash.map((h: { name: string; sha256: string }) => [h.name, h.sha256]));
|
const byName = new Map<string, string>(fork.optionsHash.map((h: { name: string; sha256: string }) => [h.name, h.sha256]));
|
||||||
const base = byName.get("heimdall_A_five_members")!;
|
const base = byName.get("heimdall_A_five_members")!;
|
||||||
assert.equal(byName.get("hidden_members_do_not_move_it"), base);
|
assert.equal(byName.get("hidden_members_do_not_move_it"), base);
|
||||||
|
const card0 = fork.card.valid[0].message as Card;
|
||||||
|
for (const raised_at of [null, "2027-01-02T03:04:05.123Z"]) {
|
||||||
|
const stamped: Card = { ...card0, raised_at };
|
||||||
|
assert.equal(optionsHash(stamped), card0.options_hash, "raised_at stays outside the hash");
|
||||||
|
}
|
||||||
|
for (const [deciders, decider_count] of [[[], 2], [card0.deciders, 64]] as const) {
|
||||||
|
const shown: Card = { ...card0, deciders: [...deciders], decider_count };
|
||||||
|
assert.equal(optionsHash(shown), card0.options_hash, "decider_count and the emptiness of deciders stay outside the hash");
|
||||||
|
}
|
||||||
assert.equal(byName.get("heimdall_E_decomposed_equals_D"), byName.get("heimdall_D_nfc_escapes_astral"), "an NFD and an NFC spelling hash alike");
|
assert.equal(byName.get("heimdall_E_decomposed_equals_D"), byName.get("heimdall_D_nfc_escapes_astral"), "an NFD and an NFC spelling hash alike");
|
||||||
for (const moved of ["heimdall_B_recommendation_changed", "heimdall_C_option_order_changed", "heimdall_F_option_id_changed", "heimdall_G_depends_emptied", "option_text_changed",
|
for (const moved of ["heimdall_B_recommendation_changed", "heimdall_C_option_order_changed", "heimdall_F_option_id_changed", "heimdall_G_depends_emptied", "option_text_changed",
|
||||||
"title_changed", "context_changed", "depends_order_changed", "depends_order_changed_reversed"]) {
|
"title_changed", "context_changed", "depends_order_changed", "depends_order_changed_reversed"]) {
|
||||||
|
|
@ -171,7 +181,7 @@ test("fork card: mutating any shown member, an option id included, changes the h
|
||||||
assert.notEqual(optionsHash(copy), base, `mutation ${i}`);
|
assert.notEqual(optionsHash(copy), base, `mutation ${i}`);
|
||||||
}
|
}
|
||||||
const hidden = structuredClone(card);
|
const hidden = structuredClone(card);
|
||||||
hidden.fork_id = "other:question:id"; hidden.leans = "yes"; hidden.deciders = ["x"]; hidden.status = "closed"; hidden.ruled_from = ["n1"];
|
hidden.fork_id = "other:question:id"; hidden.leans = "yes"; hidden.deciders = []; hidden.decider_count = 7; hidden.status = "closed"; hidden.ruled_from = ["n1"];
|
||||||
assert.equal(optionsHash(hidden), base, "fork_id and the unshown members stay outside the hash");
|
assert.equal(optionsHash(hidden), base, "fork_id and the unshown members stay outside the hash");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -195,6 +205,11 @@ test("fork card: documents validate and encode to the vector bytes", () => {
|
||||||
assert.ok(check(forkSchema, "answerResult", v.message), v.name);
|
assert.ok(check(forkSchema, "answerResult", v.message), v.name);
|
||||||
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
||||||
assert.equal(v.message.card !== null, v.name === "stale_fork", "only a stale refusal shows the new fork");
|
assert.equal(v.message.card !== null, v.name === "stale_fork", "only a stale refusal shows the new fork");
|
||||||
|
assert.deepEqual(resultRuleErrors(v.message), [], v.name);
|
||||||
|
}
|
||||||
|
for (const v of fork.resultsInvalid) {
|
||||||
|
assert.ok(!check(forkSchema, "answerResult", v.message) || resultRuleErrors(v.message).length > 0, `${v.name}: ${v.why}`);
|
||||||
|
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
||||||
}
|
}
|
||||||
for (const v of fork.unavailable) {
|
for (const v of fork.unavailable) {
|
||||||
assert.ok(check(forkSchema, "unavailable", v.message), v.name);
|
assert.ok(check(forkSchema, "unavailable", v.message), v.name);
|
||||||
|
|
@ -223,12 +238,16 @@ test("fork card: every negative is refused by the schema or a stated rule", () =
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// R-answer-result: the answer in an answer_result is the sender's own, stamped with their account, never nil (nil exists only in a record shown to a non-decider).
|
||||||
|
const resultRuleErrors = (r: { answer: { answerer: string | null } }): string[] => (r.answer.answerer === null ? ["R-answer-result: the answerer is nil"] : []);
|
||||||
|
|
||||||
// Rules a JSON Schema cannot say: a note is at most 4000 UTF-8 bytes (maxLength counts code points and is only the looser bound).
|
// Rules a JSON Schema cannot say: a note is at most 4000 UTF-8 bytes (maxLength counts code points and is only the looser bound).
|
||||||
const noteRuleErrors = (def: string, message: any): string[] => {
|
const noteRuleErrors = (def: string, message: any): string[] => {
|
||||||
const note = def === "answerRequest" ? message?.answer?.note : def === "deskAnswer" ? message?.note : null;
|
const note = def === "answerRequest" ? message?.answer?.note : def === "deskAnswer" ? message?.note : null;
|
||||||
return note != null && Buffer.byteLength(note, "utf8") > 4000 ? ["note is over 4000 UTF-8 bytes"] : [];
|
return note != null && Buffer.byteLength(note, "utf8") > 4000 ? ["note is over 4000 UTF-8 bytes"] : [];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const OP_ID = "e0d1c2b3-a495-4687-8879-6a5b4c3d2e1f";
|
||||||
const TAILS: Record<string, string> = {
|
const TAILS: Record<string, string> = {
|
||||||
unauthenticated: "a verified session is required",
|
unauthenticated: "a verified session is required",
|
||||||
body_forbidden_field: "answerer and authority are set by the receiver",
|
body_forbidden_field: "answerer and authority are set by the receiver",
|
||||||
|
|
@ -300,7 +319,7 @@ function judge(start: Card, steps: { kind: string; body: Record<string, any> | n
|
||||||
}
|
}
|
||||||
responses.push(down && reached ? unavailable() : response);
|
responses.push(down && reached ? unavailable() : response);
|
||||||
}
|
}
|
||||||
return { responses, record: { schemaVersion: "gamecult.fork.record.v0", fork_id: card.fork_id, options_hash: optionsHash(card), counted_answer_id: counted(), answers } };
|
return { responses, record: { schemaVersion: "gamecult.fork.record.v0", fork_id: card.fork_id, options_hash: optionsHash(card), counted_answer_id: counted(), answerer_count: new Set(answers.map((a) => a.answerer)).size, answers } };
|
||||||
}
|
}
|
||||||
|
|
||||||
test("fork card: every scenario replays to the vector's responses and record, bytes included", () => {
|
test("fork card: every scenario replays to the vector's responses and record, bytes included", () => {
|
||||||
|
|
@ -312,6 +331,7 @@ test("fork card: every scenario replays to the vector's responses and record, by
|
||||||
assert.deepEqual(record, sc.record, sc.name);
|
assert.deepEqual(record, sc.record, sc.name);
|
||||||
assert.ok(check(forkSchema, "record", record), sc.name);
|
assert.ok(check(forkSchema, "record", record), sc.name);
|
||||||
assert.equal(hexOf(record), sc.recordHex, sc.name);
|
assert.equal(hexOf(record), sc.recordHex, sc.name);
|
||||||
|
for (const s of sc.steps) if (s.response?.answer) assert.deepEqual(resultRuleErrors(s.response), [], sc.name);
|
||||||
for (const s of sc.steps) if (s.response) assert.ok(check(forkSchema, "answerResult", s.response) || check(forkSchema, "refusal", s.response) || check(forkSchema, "unavailable", s.response), sc.name);
|
for (const s of sc.steps) if (s.response) assert.ok(check(forkSchema, "answerResult", s.response) || check(forkSchema, "refusal", s.response) || check(forkSchema, "unavailable", s.response), sc.name);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -378,15 +398,34 @@ test("fork card: operator authority needs a cited Counted answer record", () =>
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---- transport: list, read, auth (fork-card.md "Transport") ----
|
// ---- transport: list, read, auth (fork-card.md "Transport") ----
|
||||||
const entryRuleErrors = (e: { card: Card; record: { fork_id: string; options_hash: string; counted_answer_id: string | null; answers: Record<string, any>[] } }): string[] => {
|
type Entry = { card: Card; record: { fork_id: string; options_hash: string; counted_answer_id: string | null; answerer_count: number; answers: Record<string, any>[] } };
|
||||||
|
const entryRuleErrors = (e: Entry): string[] => {
|
||||||
const errors = cardRuleErrors(e.card);
|
const errors = cardRuleErrors(e.card);
|
||||||
if (e.record.fork_id !== e.card.fork_id) errors.push("record of another fork");
|
if (e.record.fork_id !== e.card.fork_id) errors.push("record of another fork");
|
||||||
if (e.record.options_hash !== e.card.options_hash) errors.push("record under another hash");
|
if (e.record.options_hash !== e.card.options_hash) errors.push("record under another hash");
|
||||||
const counted = e.record.answers.find((a) => (a.outcome === "Counted" || a.outcome === "Revision") && a.options_hash === e.card.options_hash
|
const counted = e.record.answers.find((a) => (a.outcome === "Counted" || a.outcome === "Revision") && a.options_hash === e.card.options_hash
|
||||||
&& !e.record.answers.some((r) => r.outcome === "Retraction" && r.retracts === a.answer_id));
|
&& !e.record.answers.some((r) => r.outcome === "Retraction" && r.retracts === a.answer_id));
|
||||||
if ((counted?.answer_id ?? null) !== e.record.counted_answer_id) errors.push("counted_answer_id is not derived");
|
if ((counted?.answer_id ?? null) !== e.record.counted_answer_id) errors.push("counted_answer_id is not derived");
|
||||||
|
// R-record: named answerers are counted by distinct id, nil ones by a bound
|
||||||
|
const named = e.record.answers.filter((a) => a.answerer !== null).length, total = e.record.answers.length, n = e.record.answerer_count;
|
||||||
|
if (total === 0 && n !== 0) errors.push("R-record: no answers and a count other than 0");
|
||||||
|
else if (named === total && total > 0 && n !== new Set(e.record.answers.map((a) => a.answerer)).size) errors.push("R-record: answerer_count is not the distinct answerers");
|
||||||
|
else if (named === 0 && total > 0 && (n < 1 || n > total)) errors.push("R-record: a withheld count outside 1..answers");
|
||||||
|
// R-entry: the card's view and the record's view agree, so a withheld card never rides beside a named account
|
||||||
|
if (e.card.deciders.length > 0 && named < total) errors.push("R-entry: a decider-view card with withheld answerers");
|
||||||
|
if (e.card.deciders.length === 0 && named > 0) errors.push("R-entry: a withheld card beside a named answerer");
|
||||||
return errors;
|
return errors;
|
||||||
};
|
};
|
||||||
|
// R-list: all entries of one page are one reader's view
|
||||||
|
const viewRuleErrors = (forks: { card: Card }[]): string[] => (new Set(forks.map((f) => f.card.deciders.length > 0)).size > 1 ? ["R-list: a page mixes views"] : []);
|
||||||
|
// The non-decider twin of a decider-view entry: ids withheld, counts kept.
|
||||||
|
const withheld = (e: Entry): Entry => ({ ...e, card: { ...e.card, deciders: [] }, record: { ...e.record, answers: e.record.answers.map((a) => ({ ...a, answerer: null })) } });
|
||||||
|
const idsOf = (e: Entry): string[] => [...e.card.deciders, ...e.record.answers.map((a) => a.answerer as string)];
|
||||||
|
const bytesHaveNoId = (hex: string, ids: string[], name: string): void => {
|
||||||
|
const bytes = Buffer.from(hex, "hex");
|
||||||
|
assert.ok(ids.length > 0, name);
|
||||||
|
for (const id of ids) assert.ok(!bytes.includes(Buffer.from(id, "utf8")), `${name}: the withheld bytes carry ${id}`);
|
||||||
|
};
|
||||||
|
|
||||||
// Order is a unique rank per Mind: the refusal names both fork ids, the first holder and the second.
|
// Order is a unique rank per Mind: the refusal names both fork ids, the first holder and the second.
|
||||||
const orderErrors = (cards: { fork_id: string; order: number }[]): string[] => {
|
const orderErrors = (cards: { fork_id: string; order: number }[]): string[] => {
|
||||||
|
|
@ -465,6 +504,13 @@ test("fork transport: requests and entries validate and encode to the vector byt
|
||||||
for (const v of t[section].valid) {
|
for (const v of t[section].valid) {
|
||||||
assert.ok(check(forkSchema, def, v.message), v.name);
|
assert.ok(check(forkSchema, def, v.message), v.name);
|
||||||
if (def === "entry") assert.deepEqual(entryRuleErrors(v.message), [], v.name);
|
if (def === "entry") assert.deepEqual(entryRuleErrors(v.message), [], v.name);
|
||||||
|
if (def === "entry" && v.twin !== undefined) {
|
||||||
|
const twin = t.entry.valid.find((x: any) => x.name === v.twin);
|
||||||
|
assert.deepEqual(v.message, withheld(twin.message), `${v.name}: the twin of ${v.twin}`);
|
||||||
|
assert.equal(v.message.record.answerer_count, twin.message.record.answerer_count, "a decider and a non-decider read the same count");
|
||||||
|
bytesHaveNoId(v.messagePackHex, idsOf(twin.message), v.name);
|
||||||
|
assert.ok(twin.messagePackHex.includes(Buffer.from(idsOf(twin.message)[0], "utf8").toString("hex").toUpperCase()), "the decider view does carry the ids");
|
||||||
|
}
|
||||||
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
||||||
}
|
}
|
||||||
for (const v of t[section].invalid) {
|
for (const v of t[section].invalid) {
|
||||||
|
|
@ -473,9 +519,10 @@ test("fork transport: requests and entries validate and encode to the vector byt
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (const v of t.listInvalid) {
|
for (const v of t.listInvalid) {
|
||||||
const refused = !check(forkSchema, "list", v.message) || orderErrors(v.message.forks.map((f: any) => f.card)).length > 0;
|
const refused = !check(forkSchema, "list", v.message) || orderErrors(v.message.forks.map((f: any) => f.card)).length > 0 || viewRuleErrors(v.message.forks).length > 0;
|
||||||
assert.ok(refused, `${v.name}: ${v.why}`);
|
assert.ok(refused, `${v.name}: ${v.why}`);
|
||||||
if (v.refusal !== undefined) assert.deepEqual(orderErrors(v.message.forks.map((f: any) => f.card)), [v.refusal], v.name);
|
if (v.refusal !== undefined) assert.deepEqual(orderErrors(v.message.forks.map((f: any) => f.card)), [v.refusal], v.name);
|
||||||
|
if (v.name === "a_page_mixes_views") assert.equal(viewRuleErrors(v.message.forks).length, 1, v.name);
|
||||||
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
assert.equal(hexOf(v.message), v.messagePackHex, v.name);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -484,12 +531,26 @@ test("fork transport: every list request replays to the vector's response, newes
|
||||||
for (const sc of fork.transport.listScenarios) {
|
for (const sc of fork.transport.listScenarios) {
|
||||||
for (const e of sc.entries) assert.deepEqual(entryRuleErrors(e), [], sc.name);
|
for (const e of sc.entries) assert.deepEqual(entryRuleErrors(e), [], sc.name);
|
||||||
assert.deepEqual(orderErrors(sc.entries.map((e: any) => e.card)), [], sc.name);
|
assert.deepEqual(orderErrors(sc.entries.map((e: any) => e.card)), [], sc.name);
|
||||||
|
if (sc.twin !== undefined) {
|
||||||
|
// the non-decider page is the decider page with the ids withheld: counts, order, cursor and stamp equal, byte for byte but the withheld members
|
||||||
|
const twin = fork.transport.listScenarios.find((x: any) => x.name === sc.twin);
|
||||||
|
assert.deepEqual(sc.entries, twin.entries.map(withheld), sc.name);
|
||||||
|
assert.equal(sc.steps.length, twin.steps.length);
|
||||||
|
sc.steps.forEach((step: any, i: number) => {
|
||||||
|
const other = twin.steps[i].response;
|
||||||
|
assert.deepEqual(step.response, { ...other, forks: other.forks.map(withheld) }, sc.name);
|
||||||
|
assert.ok(twin.entries.length > 0 && step.response.forks.length > 0);
|
||||||
|
bytesHaveNoId(step.responseHex, twin.entries.flatMap(idsOf), sc.name);
|
||||||
|
});
|
||||||
|
assert.ok(twin.steps[0].responseHex.includes(Buffer.from(OP_ID, "utf8").toString("hex").toUpperCase()), "the decider page does carry the ids");
|
||||||
|
}
|
||||||
let previous: { next_cursor: string | null } | null = null;
|
let previous: { next_cursor: string | null } | null = null;
|
||||||
for (const step of sc.steps) {
|
for (const step of sc.steps) {
|
||||||
const response = judgeList(sc.entries, sc.ordinal, step.request);
|
const response = judgeList(sc.entries, sc.ordinal, step.request);
|
||||||
assert.deepEqual(response, step.response, sc.name);
|
assert.deepEqual(response, step.response, sc.name);
|
||||||
assert.ok(check(forkSchema, "list", response), sc.name);
|
assert.ok(check(forkSchema, "list", response), sc.name);
|
||||||
assert.equal(hexOf(response), step.responseHex, sc.name);
|
assert.equal(hexOf(response), step.responseHex, sc.name);
|
||||||
|
assert.deepEqual(viewRuleErrors(response.forks), [], sc.name);
|
||||||
const orders = response.forks.map((f: any) => f.card.order);
|
const orders = response.forks.map((f: any) => f.card.order);
|
||||||
assert.deepEqual(orders, [...orders].sort((a: number, b: number) => b - a), sc.name);
|
assert.deepEqual(orders, [...orders].sort((a: number, b: number) => b - a), sc.name);
|
||||||
if (previous !== null) assert.equal(step.request.cursor, previous.next_cursor, `${sc.name}: the second request follows the first page's cursor`);
|
if (previous !== null) assert.equal(step.request.cursor, previous.next_cursor, `${sc.name}: the second request follows the first page's cursor`);
|
||||||
|
|
|
||||||
|
|
@ -115,6 +115,8 @@ test("a client reconnecting from the same endpoint replaces only its own session
|
||||||
assert.throws(() => server.send(oldPeer, "schema", enc("stale")), /not connected/);
|
assert.throws(() => server.send(oldPeer, "schema", enc("stale")), /not connected/);
|
||||||
assert.equal(server.peers.length, 2);
|
assert.equal(server.peers.length, 2);
|
||||||
|
|
||||||
|
// The server accepting is not the client hearing it: send only once the client's new session is connected.
|
||||||
|
await waitFor(() => a.connection.connected, "the reconnecting client's handshake");
|
||||||
a.connection.send("schema", enc("after"));
|
a.connection.send("schema", enc("after"));
|
||||||
b.connection.send("schema", enc("still"));
|
b.connection.send("schema", enc("still"));
|
||||||
await waitFor(() => a.received.includes("re:after") && b.received.includes("re:still"), "replies after the reconnect");
|
await waitFor(() => a.received.includes("re:after") && b.received.includes("re:still"), "replies after the reconnect");
|
||||||
|
|
|
||||||
|
|
@ -173,9 +173,11 @@ def chash(c): return sha(canonical_json(c["title"], c["context"], c["options"],
|
||||||
def card(**kw):
|
def card(**kw):
|
||||||
c = {"schemaVersion": "gamecult.fork.card.v0", "fork_id": FID, "title": "Slot mark consent", "context": "Should a slot mark be shared?",
|
c = {"schemaVersion": "gamecult.fork.card.v0", "fork_id": FID, "title": "Slot mark consent", "context": "Should a slot mark be shared?",
|
||||||
"options": OPT, "recommendation": "no", "depends": DEP, "leans": None, "flag": None,
|
"options": OPT, "recommendation": "no", "depends": DEP, "leans": None, "flag": None,
|
||||||
"raised_in": {"kind": "CutSpec", "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4"}, "deciders": [OP],
|
"raised_in": {"kind": "CutSpec", "id": "cultlib-gaps:cut_spec:cut-slot-tolerance.r4"}, "raised_at": "2026-11-10T09:30:00Z", "deciders": [OP], "decider_count": 1,
|
||||||
"status": "open", "ruling": None, "ruled_from": [], "order": 3, "options_hash": ""}
|
"status": "open", "ruling": None, "ruled_from": [], "order": 3, "options_hash": ""}
|
||||||
c.update(kw); c["options_hash"] = chash(c); return c
|
c.update(kw)
|
||||||
|
if "decider_count" not in kw: c["decider_count"] = len(c["deciders"]) # the decider view: the count equals the ids; a withheld card passes its own
|
||||||
|
c["options_hash"] = chash(c); return c
|
||||||
CARD = card(); H = CARD["options_hash"]
|
CARD = card(); H = CARD["options_hash"]
|
||||||
|
|
||||||
hashes = []
|
hashes = []
|
||||||
|
|
@ -202,7 +204,7 @@ hv("title_changed", card(title="Slot mark consent?"))
|
||||||
hv("context_changed", card(context="Should slot marks be shared?"))
|
hv("context_changed", card(context="Should slot marks be shared?"))
|
||||||
hv("depends_order_changed", card(depends=DEP + ["x:question:y"]))
|
hv("depends_order_changed", card(depends=DEP + ["x:question:y"]))
|
||||||
hv("depends_order_changed_reversed", card(depends=["x:question:y"] + DEP))
|
hv("depends_order_changed_reversed", card(depends=["x:question:y"] + DEP))
|
||||||
hv("hidden_members_do_not_move_it", card(leans="yes", flag="provisional", deciders=[OP, HM], status="closed", ruling="r:1", ruled_from=["rc-1"], order=9, raised_in=None, fork_id="other:question:id"))
|
hv("hidden_members_do_not_move_it", card(leans="yes", flag="provisional", deciders=[], decider_count=2, status="closed", ruling="r:1", ruled_from=["rc-1"], order=9, raised_in=None, raised_at="2027-01-02T03:04:05.123Z", fork_id="other:question:id"))
|
||||||
hv("control_characters_and_markup_are_escaped_minimally", card(title='q " b \\ t \t n \n bell \u0007 del \u007f', context="</script> \u2028 &"))
|
hv("control_characters_and_markup_are_escaped_minimally", card(title='q " b \\ t \t n \n bell \u0007 del \u007f', context="</script> \u2028 &"))
|
||||||
hv("astral_option_id_and_text", card(options=[{"id": "\U0001F600", "text": "\U0001D11E"}, OPT[1]]))
|
hv("astral_option_id_and_text", card(options=[{"id": "\U0001F600", "text": "\U0001D11E"}, OPT[1]]))
|
||||||
assert [h["sha256"] for h in hashes if h["heimdall"] == "D"] == [h["sha256"] for h in hashes if h["heimdall"] == "E"]
|
assert [h["sha256"] for h in hashes if h["heimdall"] == "D"] == [h["sha256"] for h in hashes if h["heimdall"] == "E"]
|
||||||
|
|
@ -218,6 +220,7 @@ def body(choice=None, note=None, h=None, at="2026-11-12T19:02:00Z", fork=FID, re
|
||||||
return {"schemaVersion": "gamecult.fork.answer_request.v0", "fork_id": fork, "options_hash": h or H, "answer": a,
|
return {"schemaVersion": "gamecult.fork.answer_request.v0", "fork_id": fork, "options_hash": h or H, "answer": a,
|
||||||
"retracted": retracts is not None, "retracts": retracts, "answered_at": at, "request_id": rid or next_rid()}
|
"retracted": retracts is not None, "retracts": retracts, "answered_at": at, "request_id": rid or next_rid()}
|
||||||
def retraction(target, **kw): return body(retracts=target, **kw)
|
def retraction(target, **kw): return body(retracts=target, **kw)
|
||||||
|
NULLANS_RID = "0000b001-0000-4000-8000-00000000b001"
|
||||||
NOTE = "Keep it local: a slot mark is the author's call, and sharing it is a separate act."
|
NOTE = "Keep it local: a slot mark is the author's call, and sharing it is a separate act."
|
||||||
def srv(aid, who, at, auth="Operator"): return {"answer_id": aid, "answerer": who, "authority": auth, "received_at": at}
|
def srv(aid, who, at, auth="Operator"): return {"answer_id": aid, "answerer": who, "authority": auth, "received_at": at}
|
||||||
TAIL = {"unauthenticated": "a verified session is required", "body_forbidden_field": "answerer and authority are set by the receiver", "body_unknown_field": "the body is a closed shape",
|
TAIL = {"unauthenticated": "a verified session is required", "body_forbidden_field": "answerer and authority are set by the receiver", "body_unknown_field": "the body is a closed shape",
|
||||||
|
|
@ -237,7 +240,10 @@ def da(outcome, link, code, b, server):
|
||||||
"options_hash": b["options_hash"], "choice": a["choice"], "note": a["note"], "answered_at": b["answered_at"], "received_at": server["received_at"],
|
"options_hash": b["options_hash"], "choice": a["choice"], "note": a["note"], "answered_at": b["answered_at"], "received_at": server["received_at"],
|
||||||
"claim_in_future": future(b["answered_at"], server["received_at"]), "answerer": server["answerer"], "authority": server["authority"], "request_id": b["request_id"]}
|
"claim_in_future": future(b["answered_at"], server["received_at"]), "answerer": server["answerer"], "authority": server["authority"], "request_id": b["request_id"]}
|
||||||
def result(answer, c=None): return {"schemaVersion": "gamecult.fork.answer_result.v0", "answer": answer, "card": c, "counted_answer_id": None} # sc() fills counted_answer_id
|
def result(answer, c=None): return {"schemaVersion": "gamecult.fork.answer_result.v0", "answer": answer, "card": c, "counted_answer_id": None} # sc() fills counted_answer_id
|
||||||
def record(counted, answers, h=H, fork=FID): return {"schemaVersion": "gamecult.fork.record.v0", "fork_id": fork, "options_hash": h, "counted_answer_id": counted, "answers": answers}
|
def record(counted, answers, h=H, fork=FID, answerer_count=None):
|
||||||
|
# answerer_count: distinct answerers over every stored answer, refusals included; a withheld record passes the count it kept
|
||||||
|
n = len({a["answerer"] for a in answers}) if answerer_count is None else answerer_count
|
||||||
|
return {"schemaVersion": "gamecult.fork.record.v0", "fork_id": fork, "options_hash": h, "counted_answer_id": counted, "answerer_count": n, "answers": answers}
|
||||||
T = lambda m, sec=5: f"2026-11-12T19:{m:02d}:{sec:02d}Z"
|
T = lambda m, sec=5: f"2026-11-12T19:{m:02d}:{sec:02d}Z"
|
||||||
def step(kind, body_=None, server=None, response=None, **kw):
|
def step(kind, body_=None, server=None, response=None, **kw):
|
||||||
d = {"kind": kind, "body": body_, "server": server, "response": response}; d.update(kw); return d
|
d = {"kind": kind, "body": body_, "server": server, "response": response}; d.update(kw); return d
|
||||||
|
|
@ -385,15 +391,38 @@ def fcard(n, status="open", **kw):
|
||||||
return c
|
return c
|
||||||
def fbody(c, choice, aid_note):
|
def fbody(c, choice, aid_note):
|
||||||
return body(choice, aid_note, h=c["options_hash"], fork=c["fork_id"])
|
return body(choice, aid_note, h=c["options_hash"], fork=c["fork_id"])
|
||||||
|
def with_count(e, n): return dict(e, record=dict(e["record"], answerer_count=n))
|
||||||
|
def withheld_answers(e):
|
||||||
|
e = json.loads(json.dumps(e))
|
||||||
|
for a in e["record"]["answers"]: a["answerer"] = None
|
||||||
|
return e
|
||||||
def fentry(c, answers=(), counted=None):
|
def fentry(c, answers=(), counted=None):
|
||||||
return {"schemaVersion": "gamecult.fork.entry.v0", "card": c, "record": record(counted, list(answers), c["options_hash"], c["fork_id"])}
|
return {"schemaVersion": "gamecult.fork.entry.v0", "card": c, "record": record(counted, list(answers), c["options_hash"], c["fork_id"])}
|
||||||
|
def withheld(e):
|
||||||
|
"""The non-decider twin of a decider-view entry: ids withheld, counts kept (fork-card.md, Reader views)."""
|
||||||
|
e = json.loads(json.dumps(e)); e["card"]["deciders"] = []
|
||||||
|
for a in e["record"]["answers"]: a["answerer"] = None
|
||||||
|
return e
|
||||||
q5 = fcard(5); q4 = fcard(4); q2 = fcard(2)
|
q5 = fcard(5); q4 = fcard(4); q2 = fcard(2)
|
||||||
q3 = fcard(3, "closed", ruling="cultlib-gaps:resolution:question.q3.n1", ruled_from=["n1"]); q1 = fcard(1, "closed", ruling="cultlib-gaps:resolution:question.q1.n1")
|
q3 = fcard(3, "closed", ruling="cultlib-gaps:resolution:question.q3.n1", ruled_from=["n1"]); q1 = fcard(1, "closed", ruling="cultlib-gaps:resolution:question.q1.n1")
|
||||||
a5 = da("Counted", None, None, fbody(q5, "no", NOTE), srv("n1", OP, T(2)))
|
a5 = da("Counted", None, None, fbody(q5, "no", NOTE), srv("n1", OP, T(2)))
|
||||||
a3 = da("Counted", None, None, fbody(q3, "yes", NOTE), srv("n1", OP, T(2)))
|
a3 = da("Counted", None, None, fbody(q3, "yes", NOTE), srv("n1", OP, T(2)))
|
||||||
r3 = da("Refused", None, "fork_closed", fbody(q3, "no", NOTE), srv("n2", HM, T(4), "Steward"))
|
r3 = da("Refused", None, "fork_closed", fbody(q3, "no", NOTE), srv("n2", HM, T(4), "Steward"))
|
||||||
ENTRIES = [fentry(q5, [a5], "n1"), fentry(q4), fentry(q3, [a3, r3], "n1"), fentry(q2), fentry(q1)]
|
ENTRIES = [fentry(q5, [a5], "n1"), fentry(q4), fentry(q3, [a3, r3], "n1"), fentry(q2), fentry(q1)]
|
||||||
|
a5b = da("Revision", "n1", None, fbody(q5, "yes", NOTE), srv("n2", OP, T(4)))
|
||||||
|
TWICE = fentry(q5, [a5, a5b], "n1") # two answers, one account
|
||||||
q0 = fcard(0)
|
q0 = fcard(0)
|
||||||
|
# Revision and Retraction answers are nullable like Counted and Refused: one fork holding a Counted, its Revision and a Retraction of the Counted (appended after the older entries; nothing above moves)
|
||||||
|
q6 = fcard(6); _RID = RID[0] # the request_id counter is restored after this block: no later body changes
|
||||||
|
a6 = da("Counted", None, None, fbody(q6, "no", NOTE), srv("n1", OP, T(2)))
|
||||||
|
a6r = da("Revision", "n1", None, fbody(q6, "yes", NOTE), srv("n2", OP, T(4)))
|
||||||
|
a6x = da("Retraction", "n1", None, body(retracts="n1", h=q6["options_hash"], fork=q6["fork_id"]), srv("n3", OP, T(6)))
|
||||||
|
RID[0] = _RID
|
||||||
|
REVRET = fentry(q6, [a6, a6r, a6x], "n2")
|
||||||
|
WREVRET = withheld(REVRET)
|
||||||
|
NAMED_RETRACTION = json.loads(json.dumps(WREVRET)); NAMED_RETRACTION["record"]["answers"][2]["answerer"] = HM
|
||||||
|
WENTRIES = [withheld(e) for e in ENTRIES]
|
||||||
|
MIXED = json.loads(json.dumps(WENTRIES[2])); MIXED["record"]["answers"][0]["answerer"] = OP # one answer named among nils
|
||||||
ORDER0 = [fentry(fcard(3)), fentry(fcard(2)), fentry(fcard(1)), fentry(q0)] # a Mind whose oldest card has order 0
|
ORDER0 = [fentry(fcard(3)), fentry(fcard(2)), fentry(fcard(1)), fentry(q0)] # a Mind whose oldest card has order 0
|
||||||
q4b = dict(fcard(4), fork_id="gamecult-chat:question:q4b") # a second card on order 4 in the same Mind
|
q4b = dict(fcard(4), fork_id="gamecult-chat:question:q4b") # a second card on order 4 in the same Mind
|
||||||
def order_shared(entries):
|
def order_shared(entries):
|
||||||
|
|
@ -420,7 +449,10 @@ LISTS = [
|
||||||
{"name": "recorded_only_paged", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(status="recorded", limit=1)), lstep(ENTRIES, 412, lreq(status="recorded", cursor="c3", limit=1))]},
|
{"name": "recorded_only_paged", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(status="recorded", limit=1)), lstep(ENTRIES, 412, lreq(status="recorded", cursor="c3", limit=1))]},
|
||||||
{"name": "limit_zero_is_the_count_and_stamp_only", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(limit=0))]},
|
{"name": "limit_zero_is_the_count_and_stamp_only", "ordinal": 412, "entries": ENTRIES, "steps": [lstep(ENTRIES, 412, lreq(limit=0))]},
|
||||||
{"name": "order_zero_is_legal_but_no_cursor_names_it", "ordinal": 9, "entries": ORDER0, "steps": [lstep(ORDER0, 9, lreq(limit=3)), lstep(ORDER0, 9, lreq(cursor="c1", limit=3))]},
|
{"name": "order_zero_is_legal_but_no_cursor_names_it", "ordinal": 9, "entries": ORDER0, "steps": [lstep(ORDER0, 9, lreq(limit=3)), lstep(ORDER0, 9, lreq(cursor="c1", limit=3))]},
|
||||||
|
{"name": "non_decider_page", "ordinal": 412, "entries": WENTRIES, "twin": "all_forks_one_page", "steps": [lstep(WENTRIES, 412, lreq(limit=50))]},
|
||||||
{"name": "a_mind_with_no_forks", "ordinal": 7, "entries": [], "steps": [lstep([], 7, lreq(mind="nibu", limit=10))]},
|
{"name": "a_mind_with_no_forks", "ordinal": 7, "entries": [], "steps": [lstep([], 7, lreq(mind="nibu", limit=10))]},
|
||||||
|
{"name": "counted_revision_and_retraction_page", "ordinal": 413, "entries": [REVRET], "steps": [lstep([REVRET], 413, lreq(limit=10))]},
|
||||||
|
{"name": "non_decider_counted_revision_and_retraction_page", "ordinal": 413, "entries": [WREVRET], "twin": "counted_revision_and_retraction_page", "steps": [lstep([WREVRET], 413, lreq(limit=10))]},
|
||||||
]
|
]
|
||||||
# A page may stop short of `limit` when the next entry would pass the receiver's response budget; a list whose first
|
# A page may stop short of `limit` when the next entry would pass the receiver's response budget; a list whose first
|
||||||
# entry alone passes it is refused `fork_too_large` (422). The budget is not in the vector: a step records what the receiver answered, and the
|
# entry alone passes it is refused `fork_too_large` (422). The budget is not in the vector: a step records what the receiver answered, and the
|
||||||
|
|
@ -547,8 +579,10 @@ fork = {
|
||||||
"card": {"valid": [{"name": "operator_fork", "message": CARD, "messagePackHex": hx(CARD)},
|
"card": {"valid": [{"name": "operator_fork", "message": CARD, "messagePackHex": hx(CARD)},
|
||||||
{"name": "leans_flag_closed_two_deciders", "message": card(leans="yes", flag="provisional: Self expects this to move", deciders=[OP, HM], depends=[], raised_in=None, status="closed", ruling="x:resolution:y", ruled_from=["n1"], order=0),
|
{"name": "leans_flag_closed_two_deciders", "message": card(leans="yes", flag="provisional: Self expects this to move", deciders=[OP, HM], depends=[], raised_in=None, status="closed", ruling="x:resolution:y", ruled_from=["n1"], order=0),
|
||||||
"messagePackHex": hx(card(leans="yes", flag="provisional: Self expects this to move", deciders=[OP, HM], depends=[], raised_in=None, status="closed", ruling="x:resolution:y", ruled_from=["n1"], order=0))},
|
"messagePackHex": hx(card(leans="yes", flag="provisional: Self expects this to move", deciders=[OP, HM], depends=[], raised_in=None, status="closed", ruling="x:resolution:y", ruled_from=["n1"], order=0))},
|
||||||
|
{"name": "raised_at_nil", "message": card(raised_at=None), "messagePackHex": hx(card(raised_at=None))},
|
||||||
{"name": "depends_at_both_caps", "message": card(depends=["a"] * 7 + ["\u00e9" * 500]), "messagePackHex": hx(card(depends=["a"] * 7 + ["\u00e9" * 500]))},
|
{"name": "depends_at_both_caps", "message": card(depends=["a"] * 7 + ["\u00e9" * 500]), "messagePackHex": hx(card(depends=["a"] * 7 + ["\u00e9" * 500]))},
|
||||||
{"name": "closed_without_a_ruling_the_question_was_withdrawn", "message": card(status="closed"), "messagePackHex": hx(card(status="closed"))},
|
{"name": "closed_without_a_ruling_the_question_was_withdrawn", "message": card(status="closed"), "messagePackHex": hx(card(status="closed"))},
|
||||||
|
{"name": "withheld_deciders", "message": card(deciders=[], decider_count=2), "messagePackHex": hx(card(deciders=[], decider_count=2))},
|
||||||
{"name": "deciders_at_the_cap_of_64", "message": card(deciders=DECIDERS_64), "messagePackHex": hx(card(deciders=DECIDERS_64))}],
|
{"name": "deciders_at_the_cap_of_64", "message": card(deciders=DECIDERS_64), "messagePackHex": hx(card(deciders=DECIDERS_64))}],
|
||||||
"invalid": [bad(n, w, m) for n, w, m in [
|
"invalid": [bad(n, w, m) for n, w, m in [
|
||||||
("depends_nine_lines", "depends holds at most 8 lines (the stored question's limit)", card(depends=["a"] * 9)),
|
("depends_nine_lines", "depends holds at most 8 lines (the stored question's limit)", card(depends=["a"] * 9)),
|
||||||
|
|
@ -556,7 +590,13 @@ fork = {
|
||||||
("depends_line_1001_bytes", "rule: one byte over the limit (500 two-byte characters and one more; not expressible in the schema)", card(depends=["\u00e9" * 500 + "a"])),
|
("depends_line_1001_bytes", "rule: one byte over the limit (500 two-byte characters and one more; not expressible in the schema)", card(depends=["\u00e9" * 500 + "a"])),
|
||||||
("depends_line_over_1000_bytes", "rule: a depends line is at most 1000 UTF-8 bytes (the stored Line limit; not expressible in the schema)", card(depends=["\u00e9" * 501])),
|
("depends_line_over_1000_bytes", "rule: a depends line is at most 1000 UTF-8 bytes (the stored Line limit; not expressible in the schema)", card(depends=["\u00e9" * 501])),
|
||||||
("one_option", "two or more options", card(options=[OPT[0]], recommendation="yes")),
|
("one_option", "two or more options", card(options=[OPT[0]], recommendation="yes")),
|
||||||
("no_deciders", "a fork with no decider cannot be answered", card(deciders=[])),
|
("decider_count_zero", "a fork with no decider cannot be answered; an empty deciders list now means withheld, so the count refuses it", card(deciders=[], decider_count=0)),
|
||||||
|
("decider_count_missing", "map-mode writes every key: decider_count is required", {k: v for k, v in card().items() if k != "decider_count"}),
|
||||||
|
("decider_count_65", "decider_count is at most 64 (Bifrost's per-call list)", card(deciders=[], decider_count=65)),
|
||||||
|
("decider_count_string", "decider_count is an integer", card(deciders=[], decider_count="2")),
|
||||||
|
("decider_count_null", "decider_count is never nil: it states the count even when the ids are withheld", card(deciders=[], decider_count=None)),
|
||||||
|
("decider_count_fewer_than_the_ids", "rule R-card: a card that shows deciders shows all of them, so the count equals the ids (not expressible in the schema)", card(deciders=[OP, HM], decider_count=1)),
|
||||||
|
("decider_count_more_than_the_ids_shown", "rule R-card: a visible list must be whole; a partial list is no view (not expressible in the schema)", card(deciders=[OP, HM], decider_count=3)),
|
||||||
("null_recommendation", "every card recommends an option", dict(card(), recommendation=None)),
|
("null_recommendation", "every card recommends an option", dict(card(), recommendation=None)),
|
||||||
("unknown_key", "closed document", dict(card(), note="x")),
|
("unknown_key", "closed document", dict(card(), note="x")),
|
||||||
("option_with_a_label", "options are {id,text}; the old format's display label is not a member (fork-card.md, Old format)", card(options=[dict(OPT[0], label="Yes"), OPT[1]])),
|
("option_with_a_label", "options are {id,text}; the old format's display label is not a member (fork-card.md, Old format)", card(options=[dict(OPT[0], label="Yes"), OPT[1]])),
|
||||||
|
|
@ -565,12 +605,15 @@ fork = {
|
||||||
("duplicate_option_ids", "rule: option ids are unique (not expressible in the schema)", card(options=[OPT[0], dict(OPT[1], id=OPT[0]["id"])])),
|
("duplicate_option_ids", "rule: option ids are unique (not expressible in the schema)", card(options=[OPT[0], dict(OPT[1], id=OPT[0]["id"])])),
|
||||||
("recommendation_not_an_option", "rule: recommendation is an option id (not expressible in the schema)", card(recommendation="maybe")),
|
("recommendation_not_an_option", "rule: recommendation is an option id (not expressible in the schema)", card(recommendation="maybe")),
|
||||||
("leans_not_an_option", "rule: leans is nil or an option id (not expressible in the schema)", card(leans="maybe")),
|
("leans_not_an_option", "rule: leans is nil or an option id (not expressible in the schema)", card(leans="maybe")),
|
||||||
("duplicate_deciders", "rule: deciders are unique (not expressible in the schema)", card(deciders=[OP, OP])),
|
("duplicate_deciders", "rule: deciders are unique (not expressible in the schema)", card(deciders=[OP, OP], decider_count=2)),
|
||||||
("ruled_from_on_an_open_card", "rule: a ruling that answers a fork closes it, so an open card cites no answer in ruled_from (not expressible in the schema)", card(ruled_from=["n1"])),
|
("ruled_from_on_an_open_card", "rule: a ruling that answers a fork closes it, so an open card cites no answer in ruled_from (not expressible in the schema)", card(ruled_from=["n1"])),
|
||||||
("open_card_names_a_ruling", "rule: only a closed card names a ruling (not expressible in the schema)", card(ruling="cultlib-gaps:resolution:question.slot-mark-consent.n1")),
|
("open_card_names_a_ruling", "rule: only a closed card names a ruling (not expressible in the schema)", card(ruling="cultlib-gaps:resolution:question.slot-mark-consent.n1")),
|
||||||
("ruled_from_without_a_ruling", "rule: ruled_from lists answers a ruling was admitted from, so it needs the ruling (not expressible in the schema)", card(status="closed", ruled_from=["n1"])),
|
("ruled_from_without_a_ruling", "rule: ruled_from lists answers a ruling was admitted from, so it needs the ruling (not expressible in the schema)", card(status="closed", ruled_from=["n1"])),
|
||||||
("options_hash_of_another_card", "rule: options_hash equals the hash of the shown members (not expressible in the schema)", dict(card(), options_hash=STALE_H)),
|
("options_hash_of_another_card", "rule: options_hash equals the hash of the shown members (not expressible in the schema)", dict(card(), options_hash=STALE_H)),
|
||||||
("deciders_65", "deciders holds at most 64 (Huginn's bound)", card(deciders=DECIDERS_64 + ["00000041-0000-4000-8000-000000000041"])),
|
("raised_at_epoch_integer", "raised_at is an RFC3339 instant string or nil, not epoch milliseconds", card(raised_at=1762767000000)),
|
||||||
|
("raised_at_date_only", "raised_at is an instant with a time and a Z, not a date", card(raised_at="2026-11-10")),
|
||||||
|
("raised_at_missing", "map-mode writes every key: raised_at is required, nil when unknown", {k: v for k, v in card().items() if k != "raised_at"}),
|
||||||
|
("deciders_65", "deciders holds at most 64 (Huginn's bound)", card(deciders=DECIDERS_64 + ["00000041-0000-4000-8000-000000000041"], decider_count=65)),
|
||||||
]]},
|
]]},
|
||||||
"request": {"valid": [{"name": n, "message": m, "messagePackHex": hx(m)} for n, m in
|
"request": {"valid": [{"name": n, "message": m, "messagePackHex": hx(m)} for n, m in
|
||||||
[("choice_and_note", x1), ("choice_alone", pick), ("note_alone_clear_pick", words), ("retraction", retraction("n1")),
|
[("choice_and_note", x1), ("choice_alone", pick), ("note_alone_clear_pick", words), ("retraction", retraction("n1")),
|
||||||
|
|
@ -596,8 +639,11 @@ fork = {
|
||||||
]]},
|
]]},
|
||||||
"deskAnswer": {"valid": [{"name": n, "message": m, "messagePackHex": hx(m)} for n, m in
|
"deskAnswer": {"valid": [{"name": n, "message": m, "messagePackHex": hx(m)} for n, m in
|
||||||
[("counted", CNT), ("revision", REV), ("retraction", da("Retraction", "n1", None, retraction("n1"), sr("n8"))),
|
[("counted", CNT), ("revision", REV), ("retraction", da("Retraction", "n1", None, retraction("n1"), sr("n8"))),
|
||||||
|
("withheld_answerer", dict(CNT, answerer=None)),
|
||||||
("refused_stale", da("Refused", None, "stale_fork", stale_body(), sx1)),
|
("refused_stale", da("Refused", None, "stale_fork", stale_body(), sx1)),
|
||||||
("refused_closed_retraction_names_its_target", da("Refused", None, "fork_closed", retraction("n1"), sr("n9")))]],
|
("refused_closed_retraction_names_its_target", da("Refused", None, "fork_closed", retraction("n1"), sr("n9"))),
|
||||||
|
("withheld_answerer_revision", dict(REV, answerer=None)),
|
||||||
|
("withheld_answerer_retraction", dict(da("Retraction", "n1", None, retraction("n1", rid=NULLANS_RID), sr("n8")), answerer=None))]],
|
||||||
"invalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
"invalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
||||||
("counted_with_a_code", "only Refused carries a code", dict(CNT, code="fork_closed")),
|
("counted_with_a_code", "only Refused carries a code", dict(CNT, code="fork_closed")),
|
||||||
("refused_without_a_code", "Refused carries a reason code", dict(REF, code=None)),
|
("refused_without_a_code", "Refused carries a reason code", dict(REF, code=None)),
|
||||||
|
|
@ -612,6 +658,10 @@ fork = {
|
||||||
("outcome_lower_case", "outcome tags are Counted, Revision, Retraction, Refused", dict(CNT, outcome="counted")),
|
("outcome_lower_case", "outcome tags are Counted, Revision, Retraction, Refused", dict(CNT, outcome="counted")),
|
||||||
("no_answerer", "answerer is stamped on every stored answer", dict(CNT, answerer="")),
|
("no_answerer", "answerer is stamped on every stored answer", dict(CNT, answerer="")),
|
||||||
]]},
|
]]},
|
||||||
|
"resultsInvalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
||||||
|
("answer_result_with_a_nil_answerer", "rule R-answer-result: the answer is the sender's own, stamped with their account; a nil answerer exists only inside a record shown to a non-decider (not expressible in the schema)", dict(result(dict(CNT, answerer=None)), counted_answer_id="n1")),
|
||||||
|
("answer_result_with_a_nil_answerer_on_a_revision", "rule R-answer-result holds for every outcome: a Revision the sender just made is stamped with their account (not expressible in the schema)", dict(result(dict(REV, answerer=None)), counted_answer_id="n2")),
|
||||||
|
("answer_result_with_a_nil_answerer_on_a_retraction", "rule R-answer-result holds for every outcome: a Retraction the sender just made is stamped with their account (not expressible in the schema)", dict(result(dict(da("Retraction", "n1", None, retraction("n1", rid=NULLANS_RID), sr("n8")), answerer=None)), counted_answer_id=None))]],
|
||||||
"results": [{"name": r["name"], "httpStatus": r["httpStatus"], "message": r["message"], "messagePackHex": hx(r["message"])} for r in STORED_REFUSALS],
|
"results": [{"name": r["name"], "httpStatus": r["httpStatus"], "message": r["message"], "messagePackHex": hx(r["message"])} for r in STORED_REFUSALS],
|
||||||
"doorRefusals": [{"name": c, "httpStatus": HTTP[c], "message": refusal_doc(c), "messagePackHex": hx(refusal_doc(c))} for c in TAIL] + [{"name": "fork_too_large", "httpStatus": HTTP["fork_too_large"], "message": SHORT_PAGES[1]["steps"][0]["refusal"], "messagePackHex": SHORT_PAGES[1]["steps"][0]["refusalHex"]}],
|
"doorRefusals": [{"name": c, "httpStatus": HTTP[c], "message": refusal_doc(c), "messagePackHex": hx(refusal_doc(c))} for c in TAIL] + [{"name": "fork_too_large", "httpStatus": HTTP["fork_too_large"], "message": SHORT_PAGES[1]["steps"][0]["refusal"], "messagePackHex": SHORT_PAGES[1]["steps"][0]["refusalHex"]}],
|
||||||
"unavailable": [{"name": "second_door_down", "httpStatus": 503, "message": unavailable_doc(30), "messagePackHex": hx(unavailable_doc(30))},
|
"unavailable": [{"name": "second_door_down", "httpStatus": 503, "message": unavailable_doc(30), "messagePackHex": hx(unavailable_doc(30))},
|
||||||
|
|
@ -631,20 +681,38 @@ fork = {
|
||||||
("fork_id_empty", "a read names a fork", {"schemaVersion": "gamecult.fork.read_request.v0", "mind": "gamecult-chat", "fork_id": ""}),
|
("fork_id_empty", "a read names a fork", {"schemaVersion": "gamecult.fork.read_request.v0", "mind": "gamecult-chat", "fork_id": ""}),
|
||||||
("mind_empty", "a read names a Mind", {"schemaVersion": "gamecult.fork.read_request.v0", "mind": "", "fork_id": FID}),
|
("mind_empty", "a read names a Mind", {"schemaVersion": "gamecult.fork.read_request.v0", "mind": "", "fork_id": FID}),
|
||||||
("mind_missing", "map-mode writes every key; the Mind is the route's", {"schemaVersion": "gamecult.fork.read_request.v0", "fork_id": FID})]]},
|
("mind_missing", "map-mode writes every key; the Mind is the route's", {"schemaVersion": "gamecult.fork.read_request.v0", "fork_id": FID})]]},
|
||||||
"entry": {"valid": [{"name": n, "message": e, "messagePackHex": hx(e)} for n, e in [("open_no_answers", ENTRIES[1]), ("open_counted", ENTRIES[0]), ("closed_with_ruling_and_a_refusal", ENTRIES[2])]],
|
"entry": {"valid": [{"name": n, "message": e, "messagePackHex": hx(e)} for n, e in [("open_no_answers", ENTRIES[1]), ("open_counted", ENTRIES[0]), ("closed_with_ruling_and_a_refusal", ENTRIES[2])]]
|
||||||
|
+ [dict({"name": n, "message": e, "messagePackHex": hx(e)}, twin=t) for n, e, t in [("withheld_open_counted", WENTRIES[0], "open_counted"), ("withheld_closed_with_ruling_and_a_refusal", WENTRIES[2], "closed_with_ruling_and_a_refusal"), ("withheld_no_answers", WENTRIES[1], "open_no_answers")]]
|
||||||
|
+ [{"name": "open_counted_revision_and_retraction", "message": REVRET, "messagePackHex": hx(REVRET)},
|
||||||
|
dict({"name": "withheld_counted_revision_and_retraction", "message": WREVRET, "messagePackHex": hx(WREVRET)}, twin="open_counted_revision_and_retraction")],
|
||||||
"invalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
"invalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
||||||
("entry_without_record", "an entry holds the card and its record", {k: v for k, v in ENTRIES[0].items() if k != "record"}),
|
("entry_without_record", "an entry holds the card and its record", {k: v for k, v in ENTRIES[0].items() if k != "record"}),
|
||||||
("record_of_another_fork", "rule: the record names the card's fork (not expressible in the schema)", dict(ENTRIES[0], record=dict(ENTRIES[0]["record"], fork_id="gamecult-chat:question:q9"))),
|
("record_of_another_fork", "rule: the record names the card's fork (not expressible in the schema)", dict(ENTRIES[0], record=dict(ENTRIES[0]["record"], fork_id="gamecult-chat:question:q9"))),
|
||||||
("record_under_another_hash", "rule: the record is under the card's options_hash (not expressible in the schema)", dict(ENTRIES[0], record=dict(ENTRIES[0]["record"], options_hash=STALE_H))),
|
("record_under_another_hash", "rule: the record is under the card's options_hash (not expressible in the schema)", dict(ENTRIES[0], record=dict(ENTRIES[0]["record"], options_hash=STALE_H))),
|
||||||
("counted_answer_not_derived", "rule: counted_answer_id is the first live Counted or Revision answer under the card's hash (not expressible in the schema)", dict(ENTRIES[0], record=dict(ENTRIES[0]["record"], counted_answer_id=None))),
|
("counted_answer_not_derived", "rule: counted_answer_id is the first live Counted or Revision answer under the card's hash (not expressible in the schema)", dict(ENTRIES[0], record=dict(ENTRIES[0]["record"], counted_answer_id=None))),
|
||||||
("card_hash_wrong", "rule: the server computes options_hash and a client recomputes it; a card with another hash is no card", dict(ENTRIES[1], card=dict(ENTRIES[1]["card"], options_hash=STALE_H), record=dict(ENTRIES[1]["record"], options_hash=STALE_H)))]]},
|
("answerer_count_missing", "map-mode writes every key: answerer_count is required", dict(ENTRIES[0], record={k: v for k, v in ENTRIES[0]["record"].items() if k != "answerer_count"})),
|
||||||
|
("answerer_count_negative", "answerer_count is not negative", with_count(ENTRIES[0], -1)),
|
||||||
|
("answerer_count_null", "answerer_count is never nil: a decider and a non-decider read the same number", with_count(ENTRIES[0], None)),
|
||||||
|
("answerer_count_disagrees_with_the_ids", "rule R-record: in the decider view answerer_count is the number of distinct answerers (two here, not three; not expressible in the schema)", with_count(ENTRIES[2], 3)),
|
||||||
|
("answerer_count_counts_answers_not_people", "rule R-record: two answers by one account are one answerer (not expressible in the schema)", with_count(TWICE, 2)),
|
||||||
|
("withheld_card_with_a_named_answerer_among_nils", "rule R-entry: a withheld card beside one named answerer, even among nils (not expressible in the schema)", MIXED),
|
||||||
|
("withheld_count_over_the_answers", "rule R-record: withheld answers are at most as many answerers as answers (two answers, count 3; not expressible in the schema)", with_count(WENTRIES[2], 3)),
|
||||||
|
("withheld_count_zero_over_answers", "rule R-record: with at least one answer the count is at least 1 (not expressible in the schema)", with_count(WENTRIES[0], 0)),
|
||||||
|
("withheld_count_over_no_answers", "rule R-record: with no answers the count is 0 in both views (not expressible in the schema)", with_count(WENTRIES[1], 1)),
|
||||||
|
("decider_card_with_withheld_answers", "rule R-entry: a card that shows deciders is the decider view, so every answerer is named (not expressible in the schema)", withheld_answers(ENTRIES[0])),
|
||||||
|
("withheld_card_with_named_answers", "rule R-entry, the leak: a card with deciders withheld beside an answer that still names its account (not expressible in the schema)", dict(ENTRIES[0], card=WENTRIES[0]["card"])),
|
||||||
|
("card_hash_wrong", "rule: the server computes options_hash and a client recomputes it; a card with another hash is no card", dict(ENTRIES[1], card=dict(ENTRIES[1]["card"], options_hash=STALE_H), record=dict(ENTRIES[1]["record"], options_hash=STALE_H))),
|
||||||
|
("withheld_revision_and_retraction_beside_a_named_retraction", "rule R-entry: a withheld card beside one named answerer, a Retraction's included (not expressible in the schema)", NAMED_RETRACTION),
|
||||||
|
("withheld_revision_and_retraction_count_over_the_answers", "rule R-record: withheld answers are at most as many answerers as answers (three answers, count 4; not expressible in the schema)", with_count(WREVRET, 4))]]},
|
||||||
"listInvalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
"listInvalid": [dict(bad(n, w, m), messagePackHex=hx(m)) for n, w, m in [
|
||||||
("next_cursor_malformed", "a cursor is c<N>", dict(LISTS[0]["steps"][0]["response"], next_cursor="more")),
|
("next_cursor_malformed", "a cursor is c<N>", dict(LISTS[0]["steps"][0]["response"], next_cursor="more")),
|
||||||
("stamp_malformed", "a stamp is s<N>", dict(LISTS[0]["steps"][0]["response"], stamp="412")),
|
("stamp_malformed", "a stamp is s<N>", dict(LISTS[0]["steps"][0]["response"], stamp="412")),
|
||||||
("count_negative", "counts are not negative", dict(LISTS[0]["steps"][0]["response"], open_count=-1))]]
|
("count_negative", "counts are not negative", dict(LISTS[0]["steps"][0]["response"], open_count=-1)),
|
||||||
|
("a_page_mixes_views", "rule R-list: a page is one reader's view, so every entry agrees on whether the deciders are shown (not expressible in the schema)",
|
||||||
|
dict(LISTS[1]["steps"][0]["response"], forks=[WENTRIES[0]] + LISTS[1]["steps"][0]["response"]["forks"][1:]))]]
|
||||||
+ [dict(bad("two_cards_share_one_order", "rule: order is a unique rank per Mind; the refusal names both fork ids (not expressible in the schema)", DUP_LIST),
|
+ [dict(bad("two_cards_share_one_order", "rule: order is a unique rank per Mind; the refusal names both fork ids (not expressible in the schema)", DUP_LIST),
|
||||||
messagePackHex=hx(DUP_LIST), refusal=order_shared(DUP_LIST["forks"])[0])],
|
messagePackHex=hx(DUP_LIST), refusal=order_shared(DUP_LIST["forks"])[0])],
|
||||||
"listScenarios": [{"name": l["name"], "ordinal": l["ordinal"], "entries": l["entries"], "steps": l["steps"]} for l in LISTS],
|
"listScenarios": [dict({"name": l["name"], "ordinal": l["ordinal"], "entries": l["entries"], "steps": l["steps"]}, **({"twin": l["twin"]} if "twin" in l else {})) for l in LISTS],
|
||||||
"listShortPages": SHORT_PAGES,
|
"listShortPages": SHORT_PAGES,
|
||||||
"readTooLarge": READ_TOO_LARGE,
|
"readTooLarge": READ_TOO_LARGE,
|
||||||
"answerTooLarge": ANSWER_TOO_LARGE,
|
"answerTooLarge": ANSWER_TOO_LARGE,
|
||||||
|
|
|
||||||
|
|
@ -166,7 +166,7 @@ public static class CultDocumentMessagePackSerialization
|
||||||
|
|
||||||
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers)
|
private static (MessagePackSerializerOptions Options, IFormatterResolver[] Ahead) Compose(IFormatterResolver[] consumerResolvers)
|
||||||
{
|
{
|
||||||
// The guard goes first and wraps whatever formatter the rest of the composite would hand out, so a consumer
|
// The guard goes first and wraps whatever formatter the rest of the composite would hand out (a union, or a concrete int-keyed object for its slot count), so a consumer
|
||||||
// resolver still encodes an arm while the key check runs.
|
// resolver still encodes an arm while the key check runs.
|
||||||
var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray();
|
var ahead = consumerResolvers.Append(CultDocumentResolver.Instance).Append(CultMathResolver.Instance).Append(CultMathExprResolver.Instance).ToArray();
|
||||||
var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray());
|
var inner = CompositeResolver.Create(ahead.Append(StandardResolver.Instance).ToArray());
|
||||||
|
|
@ -214,6 +214,19 @@ public static class CultDocumentMessagePackSerialization
|
||||||
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
|
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The same decode, with the slots it read past noted (CultSlotScope). Opened around the one decode and after the depth check,
|
||||||
|
// so a payload past the nesting limit is refused, never marked.
|
||||||
|
internal static CultDecoded DeserializeMarked(Type type, byte[] payload, CultDocumentRegistry registry)
|
||||||
|
{
|
||||||
|
if (registry == null) throw new ArgumentNullException(nameof(registry));
|
||||||
|
registry.GetRequired(type);
|
||||||
|
CultPayloadDepth.Check(payload, type, PayloadDirection.Read);
|
||||||
|
using var scope = CultSlotScope.Open();
|
||||||
|
var document = MessagePackSerializer.Deserialize(type, payload, OptionsForDocument(type))
|
||||||
|
?? throw new InvalidOperationException($"MessagePack returned null for Cult document type {type.FullName}.");
|
||||||
|
return new CultDecoded(document, scope.Finding());
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Replaces slots of a serialized document payload (a MessagePack array indexed by slot) with already-encoded member
|
/// Replaces slots of a serialized document payload (a MessagePack array indexed by slot) with already-encoded member
|
||||||
/// values, leaving every other slot byte for byte as it was. A slot past the end extends the array with nil.
|
/// values, leaving every other slot byte for byte as it was. A slot past the end extends the array with nil.
|
||||||
|
|
@ -310,7 +323,7 @@ public static class CultDocumentMessagePackSerialization
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a store file: exactly one complete MessagePack array of three slots (header, schema catalog, records), or an empty
|
/// Reads a store file: exactly one complete MessagePack array of three slots (header, schema catalog, records), or an empty
|
||||||
/// array, which is an empty store. A truncated file, bytes after the array, a value that is not an array, a first slot that
|
/// array, which is an empty store. A truncated file, bytes after the array, a value that is not an array, a first slot that
|
||||||
/// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, or a body
|
/// is not a header string, a store with fewer or more slots, a record whose schema the catalog does not publish, two records under one key, or a body
|
||||||
/// that does not decode are refused with
|
/// that does not decode are refused with
|
||||||
/// <see cref="CultStoreUnreadableException"/>, so a reader never takes part of a file for the whole and never skips a slot
|
/// <see cref="CultStoreUnreadableException"/>, so a reader never takes part of a file for the whole and never skips a slot
|
||||||
/// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced.
|
/// it does not understand. The rewriting paths use this same reader as their verdict on whether a file may be replaced.
|
||||||
|
|
@ -366,6 +379,7 @@ public static class CultDocumentMessagePackSerialization
|
||||||
}
|
}
|
||||||
|
|
||||||
RequirePublishedSchemas(snapshot);
|
RequirePublishedSchemas(snapshot);
|
||||||
|
RequireUniqueKeys(snapshot);
|
||||||
return (snapshot, catalogBytes, recordBytes);
|
return (snapshot, catalogBytes, recordBytes);
|
||||||
}
|
}
|
||||||
catch (CultStoreUnreadableException)
|
catch (CultStoreUnreadableException)
|
||||||
|
|
@ -397,6 +411,17 @@ public static class CultDocumentMessagePackSerialization
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A key names one record: identity is the key string alone, compared by ordinal code units (no case folding, no normalisation).
|
||||||
|
private static void RequireUniqueKeys(CultPersistedStoreSnapshot snapshot)
|
||||||
|
{
|
||||||
|
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||||
|
foreach (var record in snapshot.Records)
|
||||||
|
{
|
||||||
|
if (!seen.Add(record.Key))
|
||||||
|
throw new CultStoreUnreadableException($"Record '{record.Key}' appears more than once.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is
|
// Slot 0 of a non-empty store, read before anything after it: a binary or nil stamp is refused as such, and a string stamp is
|
||||||
// judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read.
|
// judged by the caller's rule when it passes one. Bytes that do not open with an array and a string are left to the full read.
|
||||||
private static void JudgeStamp(byte[] payload, Action<string>? requireReadable)
|
private static void JudgeStamp(byte[] payload, Action<string>? requireReadable)
|
||||||
|
|
@ -811,4 +836,7 @@ public class SingleFileMessagePackBackingStore : SingleFileBackingStore
|
||||||
{
|
{
|
||||||
return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry);
|
return CultDocumentMessagePackSerialization.DeserializeUntyped(documentType, payload, Registry);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal override CultDecoded DeserializeMarked(Type documentType, byte[] payload) =>
|
||||||
|
CultDocumentMessagePackSerialization.DeserializeMarked(documentType, payload, Registry);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,11 +7,12 @@ namespace GameCult.Caching.MessagePack;
|
||||||
internal enum PayloadDirection { Read, Write }
|
internal enum PayloadDirection { Read, Write }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The one payload nesting count. Walks the MessagePack headers of a document payload iteratively, counting every array and
|
/// The one walk of a payload's shape: its depth and its one value. Walks the MessagePack headers of a document payload iteratively, counting every array and
|
||||||
/// map (read by a formatter or skipped past it; the outermost array is depth 1) and refusing the first container past
|
/// map (read by a formatter or skipped past it; the outermost array is depth 1) and refusing the first container past
|
||||||
/// <see cref="CultMessagePackSecurity.MaximumPayloadDepth"/>. Scalars are skipped by length, nothing is allocated per
|
/// <see cref="CultMessagePackSecurity.MaximumPayloadDepth"/>. Scalars are skipped by length, nothing is allocated per
|
||||||
/// element, and the count stack is bounded by the limit. A refusal names the type, the depth reached and the limit,
|
/// element, and the count stack is bounded by the limit. A refusal names the type, the depth reached and the limit,
|
||||||
/// never payload bytes. A payload that ends early is the deserializer's to refuse.
|
/// never payload bytes. A payload that ends early is the deserializer's to refuse; bytes after the first value are refused
|
||||||
|
/// with their count and never walked.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal static class CultPayloadDepth
|
internal static class CultPayloadDepth
|
||||||
{
|
{
|
||||||
|
|
@ -73,7 +74,13 @@ internal static class CultPayloadDepth
|
||||||
|
|
||||||
// One element is complete; close every container it was the last element of.
|
// One element is complete; close every container it was the last element of.
|
||||||
while (depth > 0 && --remaining[depth - 1] == 0) depth--;
|
while (depth > 0 && --remaining[depth - 1] == 0) depth--;
|
||||||
if (depth == 0) return;
|
if (depth == 0)
|
||||||
|
{
|
||||||
|
if (position < bytes.Length)
|
||||||
|
throw new MessagePackSerializationException(
|
||||||
|
$"{type.FullName} payload has {bytes.Length - position} bytes after its first MessagePack value (refused on {(direction == PayloadDirection.Read ? "read" : "write")}).");
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
47
src/GameCult.Caching.MessagePack/CultSlotScope.cs
Normal file
47
src/GameCult.Caching.MessagePack/CultSlotScope.cs
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
using System;
|
||||||
|
using GameCult.Caching;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.MessagePack;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The marks one payload decode noted: the guard's slot formatter reports every object whose array held more slots than its type
|
||||||
|
/// declares, with the nesting depth it sat at, and DeserializeMarked reads the finding out. Thread-bound and open only around the one
|
||||||
|
/// decode, so a decode outside it (a CultMesh read, a codec) notes nothing.
|
||||||
|
/// </summary>
|
||||||
|
internal sealed class CultSlotScope : IDisposable
|
||||||
|
{
|
||||||
|
[ThreadStatic] private static CultSlotScope? current;
|
||||||
|
|
||||||
|
private readonly CultSlotScope? previous;
|
||||||
|
private string? typeName;
|
||||||
|
private int slotCount;
|
||||||
|
private int declaredCount;
|
||||||
|
private int occurrences;
|
||||||
|
private bool allAtDepthZero = true;
|
||||||
|
|
||||||
|
private CultSlotScope() => previous = current;
|
||||||
|
|
||||||
|
internal static CultSlotScope? Current => current;
|
||||||
|
|
||||||
|
internal static CultSlotScope Open() => current = new CultSlotScope();
|
||||||
|
|
||||||
|
// 0 for the document's own object; each object nested inside it adds one (a union adds none, its arm is the object).
|
||||||
|
internal int Depth { get; set; }
|
||||||
|
|
||||||
|
internal void Note(string name, int slots, int declared)
|
||||||
|
{
|
||||||
|
if (occurrences++ == 0)
|
||||||
|
{
|
||||||
|
typeName = name;
|
||||||
|
slotCount = slots;
|
||||||
|
declaredCount = declared;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Depth > 0) allAtDepthZero = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal CultUnreadSlots? Finding() =>
|
||||||
|
occurrences == 0 ? null : new CultUnreadSlots(typeName!, slotCount, declaredCount, occurrences, allAtDepthZero, explained: false);
|
||||||
|
|
||||||
|
public void Dispose() => current = previous;
|
||||||
|
}
|
||||||
|
|
@ -1,14 +1,17 @@
|
||||||
using System;
|
using System;
|
||||||
|
using System.Buffers;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
using MessagePack;
|
using MessagePack;
|
||||||
using MessagePack.Formatters;
|
using MessagePack.Formatters;
|
||||||
|
using MessagePack.Resolvers;
|
||||||
|
|
||||||
namespace GameCult.Caching.MessagePack;
|
namespace GameCult.Caching.MessagePack;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The one place an unknown union arm or an unnamed subtype is decided. A union is an abstract class or interface carrying
|
/// The one sensor of an unknown union arm, an unnamed subtype and a slot past the reader's declared slots. A union is an abstract class or interface carrying
|
||||||
/// its own [Union] attributes (never inherited: a union whose arms are also a union has its own key set). MessagePack-CSharp
|
/// its own [Union] attributes (never inherited: a union whose arms are also a union has its own key set). MessagePack-CSharp
|
||||||
/// still encodes the [key, armSlots] shape; this wraps its formatter so that a key the union does not declare is refused on
|
/// still encodes the [key, armSlots] shape; this wraps its formatter so that a key the union does not declare is refused on
|
||||||
/// read, and a value whose runtime type is not a declared arm is refused on write, where MessagePack alone yields null and nil.
|
/// read, and a value whose runtime type is not a declared arm is refused on write, where MessagePack alone yields null and nil.
|
||||||
|
|
@ -18,11 +21,117 @@ internal sealed class CultUnionGuard(IFormatterResolver inner) : IFormatterResol
|
||||||
public IMessagePackFormatter<T>? GetFormatter<T>()
|
public IMessagePackFormatter<T>? GetFormatter<T>()
|
||||||
{
|
{
|
||||||
var arms = typeof(T).GetCustomAttributes<UnionAttribute>(inherit: false).ToArray();
|
var arms = typeof(T).GetCustomAttributes<UnionAttribute>(inherit: false).ToArray();
|
||||||
return arms.Length == 0 || inner.GetFormatter<T>() is not { } formatter ? null : new Formatter<T>(formatter, arms);
|
if (arms.Length == 0)
|
||||||
|
return inner.GetFormatter<T>() is { } objectFormatter && KeyLayout(objectFormatter) ? new SlotFormatter<T>(objectFormatter) : null;
|
||||||
|
return inner.GetFormatter<T>() is not { } formatter ? null : new Formatter<T>(formatter, arms);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class Formatter<T>(IMessagePackFormatter<T> inner, UnionAttribute[] arms) : IMessagePackFormatter<T>
|
// The formatter that writes and reads the bytes: the one a guard wrapper delegates to, else the formatter itself. A check that a
|
||||||
|
// generated formatter is the one in use asks this (tests/GameCult.Caching.GeneratorConsumer).
|
||||||
|
internal static object Inner(object formatter) => formatter is IWrapper wrapper ? wrapper.Inner : formatter;
|
||||||
|
|
||||||
|
private interface IWrapper
|
||||||
{
|
{
|
||||||
|
object Inner { get; }
|
||||||
|
}
|
||||||
|
|
||||||
|
// The one decision of whether a type is measured: the formatter the chain hands out for an int-keyed [MessagePackObject] type is
|
||||||
|
// MessagePack's own [Key]-layout formatter, the very instance StandardResolver (or its AllowPrivate twin) builds for it. A formatter a
|
||||||
|
// consumer resolver or a [MessagePackFormatter] attribute supplies owns its layout and its tolerance, and a [MessagePackObject]
|
||||||
|
// collection is written as its items by the generic resolver: none of those is measured. Identity, as ResolverClaim asks it.
|
||||||
|
private static bool KeyLayout<T>(IMessagePackFormatter<T> formatter)
|
||||||
|
{
|
||||||
|
if (typeof(T).IsAbstract || typeof(T).IsInterface ||
|
||||||
|
typeof(T).GetCustomAttribute<MessagePackObjectAttribute>(inherit: true) is not { KeyAsPropertyName: false })
|
||||||
|
return false;
|
||||||
|
if (ReferenceEquals(formatter, Of<T>(AttributeFormatterResolver.Instance)) || ReferenceEquals(formatter, Of<T>(DynamicGenericResolver.Instance)))
|
||||||
|
return false;
|
||||||
|
return ReferenceEquals(formatter, Of<T>(StandardResolver.Instance)) || ReferenceEquals(formatter, Of<T>(StandardResolverAllowPrivate.Instance));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IMessagePackFormatter<T>? Of<T>(IFormatterResolver resolver)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return resolver.GetFormatter<T>();
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Counts a concrete object's array header against its declared slots and notes the excess into the open CultSlotScope, then lets the
|
||||||
|
// inner formatter read it unchanged: no byte scan, no second decode. With no scope open it is a depth-free pass-through.
|
||||||
|
// Declared slots are what the type's own formatter writes: the array header of one instance written through it, measured on the
|
||||||
|
// first read inside a scope (not in GetFormatter, so a self-referential type never recurses into its own resolver cache) and kept.
|
||||||
|
private sealed class SlotFormatter<T>(IMessagePackFormatter<T> inner) : IMessagePackFormatter<T>, IWrapper
|
||||||
|
{
|
||||||
|
object IWrapper.Inner => inner;
|
||||||
|
|
||||||
|
private const int NotMeasured = -1;
|
||||||
|
|
||||||
|
// A type no instance of which can be written is not guarded: no count can exceed this.
|
||||||
|
private const int Unguarded = int.MaxValue;
|
||||||
|
|
||||||
|
private int declared = NotMeasured;
|
||||||
|
|
||||||
|
public void Serialize(ref MessagePackWriter writer, T value, MessagePackSerializerOptions options) =>
|
||||||
|
inner.Serialize(ref writer, value, options);
|
||||||
|
|
||||||
|
public T Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
|
||||||
|
{
|
||||||
|
var scope = CultSlotScope.Current;
|
||||||
|
if (scope == null) return inner.Deserialize(ref reader, options);
|
||||||
|
var peek = reader.CreatePeekReader();
|
||||||
|
if (!peek.TryReadNil() && peek.NextMessagePackType == MessagePackType.Array)
|
||||||
|
{
|
||||||
|
var count = peek.ReadArrayHeader();
|
||||||
|
if (declared == NotMeasured) declared = Measure(options);
|
||||||
|
if (count > declared) scope.Note(typeof(T).Name, count, declared);
|
||||||
|
}
|
||||||
|
|
||||||
|
scope.Depth++;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return inner.Deserialize(ref reader, options);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
scope.Depth--;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The uninitialized instance, else one from a parameterless constructor (public or not) when that write throws.
|
||||||
|
private int Measure(MessagePackSerializerOptions options)
|
||||||
|
{
|
||||||
|
if (Written(() => (T)RuntimeHelpers.GetUninitializedObject(typeof(T)), options) is { } slots) return slots;
|
||||||
|
var constructor = typeof(T).GetConstructor(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, null, Type.EmptyTypes, null);
|
||||||
|
return constructor != null && Written(() => (T)constructor.Invoke(null), options) is { } constructed ? constructed : Unguarded;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int? Written(Func<T> instance, MessagePackSerializerOptions options)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var buffer = new ArrayBufferWriter<byte>();
|
||||||
|
var writer = new MessagePackWriter(buffer);
|
||||||
|
inner.Serialize(ref writer, instance(), options);
|
||||||
|
writer.Flush();
|
||||||
|
var written = new MessagePackReader(buffer.WrittenMemory);
|
||||||
|
return written.NextMessagePackType == MessagePackType.Array ? written.ReadArrayHeader() : null;
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class Formatter<T>(IMessagePackFormatter<T> inner, UnionAttribute[] arms) : IMessagePackFormatter<T>, IWrapper
|
||||||
|
{
|
||||||
|
object IWrapper.Inner => inner;
|
||||||
|
|
||||||
private readonly HashSet<int> keys = arms.Select(arm => arm.Key).ToHashSet();
|
private readonly HashSet<int> keys = arms.Select(arm => arm.Key).ToHashSet();
|
||||||
private readonly HashSet<Type> types = arms.Select(arm => arm.SubType).ToHashSet();
|
private readonly HashSet<Type> types = arms.Select(arm => arm.SubType).ToHashSet();
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -636,7 +636,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
||||||
record,
|
record,
|
||||||
VersionOf(metadata.Payload),
|
VersionOf(metadata.Payload),
|
||||||
catalog,
|
catalog,
|
||||||
(type, payload) => CultDocumentMessagePackSerialization.DeserializeUntyped(type, payload, Registry),
|
(type, payload) => CultDocumentMessagePackSerialization.DeserializeMarked(type, payload, Registry),
|
||||||
out report);
|
out report);
|
||||||
// The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with.
|
// The manifest is this store's record of what is on disk: the stored id and the version a commit condition compares with.
|
||||||
stored.StoredSchemaId = metadata.SchemaId;
|
stored.StoredSchemaId = metadata.SchemaId;
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@
|
||||||
<PackageReference Include="System.Text.Json" Version="9.0.10" />
|
<PackageReference Include="System.Text.Json" Version="9.0.10" />
|
||||||
<PackageReference Include="MessagePackAnalyzer" Version="3.1.7" PrivateAssets="all" />
|
<PackageReference Include="MessagePackAnalyzer" Version="3.1.7" PrivateAssets="all" />
|
||||||
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
|
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
|
||||||
|
<InternalsVisibleTo Include="GameCult.Caching.GeneratorConsumer" />
|
||||||
<None Include="buildTransitive\GameCult.Caching.MessagePack.targets" Pack="true" PackagePath="buildTransitive\" />
|
<None Include="buildTransitive\GameCult.Caching.MessagePack.targets" Pack="true" PackagePath="buildTransitive\" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,10 @@ await cache.PullAllBackingStoresAsync();
|
||||||
- The raw CultNet document lane should treat these payload bytes as already
|
- The raw CultNet document lane should treat these payload bytes as already
|
||||||
blessed, not decode and re-encode them for sport.
|
blessed, not decode and re-encode them for sport.
|
||||||
|
|
||||||
|
## Store format and rollback
|
||||||
|
|
||||||
|
A store whose elements hold an id is marked `cultcache.store.v3` (a directory store `v5`); the single-file and directory stores here write the mark from content. See the Store format and rollback section of the GameCult.Caching README. Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case. The only carry-back is narrow and lossy (a whole-store write that leaves no id in the store drops the mark and the ids); otherwise restore from a copy taken before upgrading, and upgrade every reader before any writer.
|
||||||
|
|
||||||
## MessagePack source generator
|
## MessagePack source generator
|
||||||
|
|
||||||
NuGet consumers get `[assembly: MessagePackAssumedFormattable(typeof(CultRecordRef<>))]` from the package, so a
|
NuGet consumers get `[assembly: MessagePackAssumedFormattable(typeof(CultRecordRef<>))]` from the package, so a
|
||||||
|
|
|
||||||
|
|
@ -202,6 +202,20 @@ flatten, remove, commit) is refused with the typed schema conflict before anythi
|
||||||
Declaring its id compatible on any registered type makes it writable and is the consent to the loss:
|
Declaring its id compatible on any registered type makes it writable and is the consent to the loss:
|
||||||
the next write stores it under the writing type's own id and sheds the slots.
|
the next write stores it under the writing type's own id and sheds the slots.
|
||||||
|
|
||||||
|
A record whose payload carries a slot past the slots its reader declares, at any depth (nested object, list
|
||||||
|
element, union arm, unit arm), is read-only the same way, whatever id it is stored under. The one consent
|
||||||
|
is the declaration above, and it reaches only depth-0 slots the schema comparison reports ignored under a
|
||||||
|
listed compatible id, up to the highest slot it reports: a mark anywhere else, past that slot, or at depth 0
|
||||||
|
under the reader's own id, is not consented to. Declared slots are what the type's own formatter writes; only
|
||||||
|
a type MessagePack writes by its `[Key]` layout is checked, and one no instance of which can be written is not
|
||||||
|
(`cultlib-gaps:ruling:slot-unmeasurable-type`; see the schema-compatibility contract for both exemptions and
|
||||||
|
for the user code the measure runs).
|
||||||
|
A flatten of a variant whose base chain holds a record that is not owned is refused with that record's
|
||||||
|
refusal, because it would write the base's decoded view; the variant's own writes persist only its delta and
|
||||||
|
stay allowed. A plain write to another key of a value read from a read-only record is a copy this rule does
|
||||||
|
not police. `CultCache.IsWritable(key)` answers what the staging refusal answers for a removal or a write of
|
||||||
|
the record's type, read-only and missing home stores included; it does not predict that flatten.
|
||||||
|
|
||||||
The rule is checked again where the write lands, under the store's lock: a write or removal at a key
|
The rule is checked again where the write lands, under the store's lock: a write or removal at a key
|
||||||
whose durable record is not owned, because another writer rewrote it after this cache
|
whose durable record is not owned, because another writer rewrote it after this cache
|
||||||
loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can
|
loaded, is refused typed, and the record is reloaded as the store holds it, read-only when a type can
|
||||||
|
|
@ -389,6 +403,8 @@ runtime enforces the same bound on read and on write, and the refusal names the
|
||||||
the depth and the limit, never payload bytes. The vectors are in
|
the depth and the limit, never payload bytes. The vectors are in
|
||||||
`tests/vectors/union-parity` (`bounds-outcomes.msgpack`).
|
`tests/vectors/union-parity` (`bounds-outcomes.msgpack`).
|
||||||
|
|
||||||
|
A record's payload is exactly one MessagePack value: every runtime refuses a payload with bytes after its first value wherever it reads a payload under the nesting bound, and on write where a write takes bytes the library did not encode itself (a C# type's own formatter, a Rust raw envelope), without walking those bytes, so a tail past the limit is refused as bytes after the value and not as depth; the refusal names the type and the count of bytes after the value, never the bytes (vectors: `tests/vectors/union-parity/bytes-after-outcomes.msgpack`). A CDN manifest is stored and keyed as a record and follows the same rule (`contracts/cultmesh/content-acceptance-vectors.json`: `trailing_nil`, `trailing_reserved_code` and `trailing_truncated_array` are refused). Streams of values (CultNet frames, a store's record array) are different contracts and are not covered.
|
||||||
|
|
||||||
## Variant Records
|
## Variant Records
|
||||||
|
|
||||||
A variant record is a record whose document is a base record's document with some
|
A variant record is a record whose document is a base record's document with some
|
||||||
|
|
@ -535,15 +551,14 @@ Read directly when:
|
||||||
Allowed when:
|
Allowed when:
|
||||||
|
|
||||||
- persisted schema has missing slots that the local schema can default
|
- persisted schema has missing slots that the local schema can default
|
||||||
- persisted schema has extra slots the local reader can safely ignore or preserve
|
|
||||||
- slot meanings remain compatible
|
- slot meanings remain compatible
|
||||||
|
|
||||||
Soft migration should emit warnings, not silent shrugs.
|
A persisted schema with extra slots the local reader lacks is not soft: the record is read-only (see the ownership
|
||||||
|
rule above). Soft migration should emit warnings, not silent shrugs.
|
||||||
|
|
||||||
Typical warnings:
|
Typical warnings:
|
||||||
|
|
||||||
- missing slot defaulted
|
- missing slot defaulted
|
||||||
- unknown extra slot ignored
|
|
||||||
- nullable/default behavior widened
|
- nullable/default behavior widened
|
||||||
- field preserved but not surfaced natively
|
- field preserved but not surfaced natively
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -107,10 +107,48 @@ persisted slots honestly:
|
||||||
|
|
||||||
- persisted schema id differs, but the embedded catalog points to a compatible
|
- persisted schema id differs, but the embedded catalog points to a compatible
|
||||||
local schema id
|
local schema id
|
||||||
- persisted slot is missing locally and can be ignored
|
- persisted slot is missing locally and the record is read-only until its id is declared compatible
|
||||||
|
(see Extra slots are read-only, below)
|
||||||
- local slot is missing in persisted data and can fall back to the local
|
- local slot is missing in persisted data and can fall back to the local
|
||||||
default value
|
default value
|
||||||
|
|
||||||
|
Extra slots are read-only. A record whose payload carries a slot past the slots its reader declares, at any
|
||||||
|
depth (the record, a nested object, a list element, a union arm, a unit arm), is loaded read-only: it is
|
||||||
|
served, `CultStoredDocument.UnreadSlots` names the first type read with extra slots and both counts, and
|
||||||
|
`CultCache.IsWritable(key)` is false. A staged write or removal of it is refused with
|
||||||
|
`CultSchemaConflictException` (`UnreadSlots` set; the message names the type and the counts, never a slot
|
||||||
|
value). A missing trailing slot keeps its default and stays writable; a non-nil value in a retired gap below
|
||||||
|
the highest key is not extra. Declaring an id compatible consents only to depth-0 slots the schema comparison
|
||||||
|
reports ignored under that id, and only up to the highest slot it reports: a payload carrying a slot past that
|
||||||
|
one is not consented to. Any other mark (nested, list element, arm, unit arm, or depth 0 under the reader's
|
||||||
|
own id) keeps the record read-only whatever is declared.
|
||||||
|
|
||||||
|
A type's declared slots are the array its own formatter writes for one instance, never a reflected member
|
||||||
|
list, so a non-public `[Key]` member of an `AllowPrivate` type counts. Only a concrete int-keyed
|
||||||
|
`[MessagePackObject]` type whose formatter is MessagePack's own `[Key]`-layout formatter is checked. Not
|
||||||
|
checked, and so read past with any extra slots dropped on the next write, as before:
|
||||||
|
|
||||||
|
- a type formatted by anything else: its own `[MessagePackFormatter]`, a formatter of its own that a
|
||||||
|
`CultCacheFormatterResolver` (or any resolver ahead of MessagePack's) hands out, or the collection formatter
|
||||||
|
of a `[MessagePackObject]` collection type such as a `List<T>` subclass. Such a formatter owns its layout and
|
||||||
|
its tolerance. A resolver that hands out MessagePack's own `[Key]`-layout formatter (StandardResolver's, or
|
||||||
|
StandardResolverAllowPrivate's) does not exempt the type: it is checked and marks;
|
||||||
|
- a type no instance of which can be written: no write of an uninitialized instance and no write of one
|
||||||
|
from a parameterless constructor succeeds (for example a `[Key]` getter that throws on an uninitialized
|
||||||
|
instance and no parameterless constructor). This gap is ruled: `cultlib-gaps:ruling:slot-unmeasurable-type`.
|
||||||
|
|
||||||
|
Measuring runs user code once per type per process, on the first read of the type in a store load: the
|
||||||
|
type's `[Key]` getters on an uninitialized instance and, if that write throws, its parameterless
|
||||||
|
constructor (public or not) and the getters on that instance. Any failure of both leaves the type unchecked
|
||||||
|
for the rest of the process, and nothing else: it does not fail the read.
|
||||||
|
|
||||||
|
`Flatten` of a variant writes its base chain's decoded view into a plain record, so it is refused when any
|
||||||
|
record on that chain is not owned by its home store (a mark nothing explains, or undeclared drift that drops
|
||||||
|
slots), with that record's own refusal naming its key. Writing or removing the variant itself stays allowed:
|
||||||
|
it persists only its delta. `CultCache.IsWritable(key)` is the write's own answer for a removal or a write of
|
||||||
|
the record's type: it is also false when the record's home store is read-only or no attached store is home to
|
||||||
|
its type. The one write it does not predict is that flatten.
|
||||||
|
|
||||||
The cache emits a typed migration report with:
|
The cache emits a typed migration report with:
|
||||||
|
|
||||||
- exact vs compatible-drift classification
|
- exact vs compatible-drift classification
|
||||||
|
|
|
||||||
|
|
@ -516,14 +516,22 @@ namespace GameCult.Caching
|
||||||
// resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it).
|
// resolved to Descriptor, until a write of this record puts it under Descriptor's id (a write of an owned record moves it).
|
||||||
public string StoredSchemaId { get; internal set; }
|
public string StoredSchemaId { get; internal set; }
|
||||||
|
|
||||||
// True when the schema this record was loaded under left none of its persisted slots unread by Descriptor, so the type reads
|
// True when the schema this record was loaded under left none of its persisted slots unread by Descriptor and its payload
|
||||||
// it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots,
|
// carried none past Descriptor's, so the type reads it whole and owns it (CultDocumentRegistry.Owns). Set by ToStoredDocument from the schema comparison's ignored slots,
|
||||||
// before any variant override is shed. A record this cache constructed or wrote is stored under Descriptor's own id, which
|
// before any variant override is shed. A record this cache constructed or wrote is stored under Descriptor's own id, which
|
||||||
// the registry declares, so only a loaded record under another id is ever asked. The staging half asks it;
|
// the registry declares, so only a loaded record under another id is ever asked. The staging half asks it;
|
||||||
// ConditionsHold never reads it. Only the store's loaded view carries it: the cache's resolved copies and flattens are never
|
// ConditionsHold never reads it. Only the store's loaded view carries it: the cache's resolved copies and flattens are never
|
||||||
// asked.
|
// asked.
|
||||||
internal bool StoredSchemaReadWhole { get; set; }
|
internal bool StoredSchemaReadWhole { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What the store's decode of this record's payload found past the slots this build declares, at any depth; null when none. Set by
|
||||||
|
/// ToStoredDocument on the store's loaded view only (the cache's resolved copy of a variant does not carry it; ask
|
||||||
|
/// <see cref="CultCache.IsWritable"/>). A mark that is not <see cref="CultUnreadSlots.Explained"/> keeps the record
|
||||||
|
/// unowned even under a declared id: it cannot be written or removed (HeldOwned).
|
||||||
|
/// </summary>
|
||||||
|
public CultUnreadSlots? UnreadSlots { get; internal set; }
|
||||||
|
|
||||||
// The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for,
|
// The catalog entry that publishes StoredSchemaId in the store this record was loaded from, with the id it was published for,
|
||||||
// so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record
|
// so staging can ask WrittenOwns the question the store's judge asks, over the catalog the record came with. Null for a record
|
||||||
// this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A write that
|
// this cache constructed or wrote, which is stored under Descriptor's own id and published by Descriptor's entry. A write that
|
||||||
|
|
@ -2300,8 +2308,9 @@ namespace GameCult.Caching
|
||||||
|
|
||||||
// The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in
|
// The one-shot rewrite for stores written before element ids existed: upserts every record whose ids exist only in
|
||||||
// memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records
|
// memory (minted when it loaded), through the normal write path, one commit per home store. Returns how many records
|
||||||
// it rewrote; a second call finds none. A selected record that is neither declared nor read whole is read-only,
|
// it rewrote; a second call finds none. A selected record that is not owned (neither declared nor read whole, or held
|
||||||
// so its home store's commit is refused with CultSchemaConflictException; commits to earlier stores stand.
|
// read-only for slots past this build's, see CultStoredDocument.UnreadSlots) is read-only, so its home store's commit is
|
||||||
|
// refused with CultSchemaConflictException until a build that reads those slots rewrites it; commits to earlier stores stand.
|
||||||
public int MintElementIds()
|
public int MintElementIds()
|
||||||
{
|
{
|
||||||
var groups = Held(() => _entries.Values
|
var groups = Held(() => _entries.Values
|
||||||
|
|
@ -2656,6 +2665,22 @@ namespace GameCult.Caching
|
||||||
return _entries.TryGetValue(key.Value, out var stored) ? stored : null;
|
return _entries.TryGetValue(key.Value, out var stored) ? stored : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Whether a removal of the key, or a write of the record's own type, would be let through: the write's own decision, asked of
|
||||||
|
/// the record the cache holds. False when no attached store is home to the record's type, when its home store is read-only, or
|
||||||
|
/// when that store holds it unowned (undeclared drift that drops persisted slots, or a payload carrying slots past this build's,
|
||||||
|
/// see <see cref="CultStoredDocument.UnreadSlots"/>). For a key the cache does not hold, the removal check every store makes of it
|
||||||
|
/// (false when a store lists it foreign). One write it does not predict: a Flatten of a variant whose base chain holds an unowned record is refused, though
|
||||||
|
/// the variant itself is writable. Live and advisory: the write still decides.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsWritable(CultRecordKey key)
|
||||||
|
{
|
||||||
|
lock (Gate())
|
||||||
|
return _entries.TryGetValue(key.Value, out var held)
|
||||||
|
? WriteRefusal(held, upsert: false, null, out _) == null
|
||||||
|
: _stores.All(entry => entry.Store.Refusal(key.Value, null) == null);
|
||||||
|
}
|
||||||
|
|
||||||
public object? Get(CultRecordKey key)
|
public object? Get(CultRecordKey key)
|
||||||
{
|
{
|
||||||
lock (Gate())
|
lock (Gate())
|
||||||
|
|
@ -3151,6 +3176,8 @@ namespace GameCult.Caching
|
||||||
IReadOnlyCollection<string>? removed)
|
IReadOnlyCollection<string>? removed)
|
||||||
{
|
{
|
||||||
var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet<string>(StringComparer.Ordinal);
|
var removedKeys = removed?.ToHashSet(StringComparer.Ordinal) ?? new HashSet<string>(StringComparer.Ordinal);
|
||||||
|
var batch = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
|
||||||
|
foreach (var stored in admitted) batch[stored.Key.Value] = stored;
|
||||||
var homes = new List<CacheBackingStore>();
|
var homes = new List<CacheBackingStore>();
|
||||||
var globals = new Dictionary<Type, string>(_globals);
|
var globals = new Dictionary<Type, string>(_globals);
|
||||||
var evictedKeys = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal);
|
var evictedKeys = evicted.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal);
|
||||||
|
|
@ -3199,23 +3226,57 @@ namespace GameCult.Caching
|
||||||
|
|
||||||
void Claim(CultStoredDocument stored, bool upsert)
|
void Claim(CultStoredDocument stored, bool upsert)
|
||||||
{
|
{
|
||||||
var home = Home(stored.Descriptor.DocumentType);
|
if (WriteRefusal(stored, upsert, batch, out var home) is { } refusal) throw refusal;
|
||||||
if (home == null)
|
if (home != null) homes.Add(home);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The one answer to whether a record may be written (upsert: a write of its type) or its key removed, before anything is staged.
|
||||||
|
// Claim throws it; IsWritable asks it of the held record's removal form. Null when the write may land; home is where it lands,
|
||||||
|
// null for a store-less cache. batch holds the records the same batch admits, by key.
|
||||||
|
private Exception? WriteRefusal(CultStoredDocument stored, bool upsert, IReadOnlyDictionary<string, CultStoredDocument>? batch, out CacheBackingStore? home)
|
||||||
|
{
|
||||||
|
home = Home(stored.Descriptor.DocumentType);
|
||||||
|
if (home == null)
|
||||||
|
{
|
||||||
|
if (_stores.Count > 0)
|
||||||
|
return new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}.");
|
||||||
|
// No store, no file to judge, but the same rule: a record is replaced only by its own type's write.
|
||||||
|
return upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor)
|
||||||
|
? CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor)
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (home.IsReadOnly)
|
||||||
|
return new InvalidOperationException(
|
||||||
|
$"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}.");
|
||||||
|
return home.Refusal(stored.Key.Value, upsert ? stored.Descriptor : null)
|
||||||
|
?? (upsert && stored.Flattens ? FlattenRefusal(stored, batch) : null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A flatten writes its base chain's decoded view into a plain record, so it is refused when a record on that chain is not owned
|
||||||
|
// by its home store (the slots or drift that record's decode dropped would be lost): that ancestor's own refusal. The chain starts
|
||||||
|
// at the staged delta's base when the variant is flattened as it lands, else at the held variant's; a key the batch admits is
|
||||||
|
// judged by its own claim, and the walk follows its staged form.
|
||||||
|
private CultSchemaConflictException? FlattenRefusal(CultStoredDocument flatten, IReadOnlyDictionary<string, CultStoredDocument>? batch)
|
||||||
|
{
|
||||||
|
var delta = flatten.Variant ?? (_entries.TryGetValue(flatten.Key.Value, out var variant) ? variant.Variant : null);
|
||||||
|
var walked = new HashSet<string>(StringComparer.Ordinal) { flatten.Key.Value };
|
||||||
|
for (var key = delta?.BaseKey; key != null && walked.Add(key);)
|
||||||
|
{
|
||||||
|
CultStoredDocument? ancestor;
|
||||||
|
if (batch == null || !batch.TryGetValue(key, out ancestor))
|
||||||
{
|
{
|
||||||
if (_stores.Count > 0)
|
if (!_entries.TryGetValue(key, out ancestor))
|
||||||
throw new InvalidOperationException($"No backing store is home to {stored.Descriptor.SchemaName}.");
|
return null;
|
||||||
// No store, no file to judge, but the same rule: a record is replaced only by its own type's write.
|
if (Home(ancestor.Descriptor.DocumentType)?.Refusal(key, null) is { } refusal)
|
||||||
if (upsert && _entries.TryGetValue(stored.Key.Value, out var inMemory) && !CacheBackingStore.UpsertOwned(inMemory, stored.Descriptor))
|
return refusal;
|
||||||
throw CacheBackingStore.Retyped(inMemory, stored.Key.Value, stored.Descriptor);
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (home.IsReadOnly)
|
key = ancestor.Variant?.BaseKey;
|
||||||
throw new InvalidOperationException(
|
|
||||||
$"{stored.Descriptor.SchemaName} record {stored.Key.Value} routes to read-only backing store {home}.");
|
|
||||||
home.RefuseUndeclared(stored.Key.Value, upsert ? stored.Descriptor : null);
|
|
||||||
homes.Add(home);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class VariantPlan
|
private sealed class VariantPlan
|
||||||
|
|
@ -3495,10 +3556,11 @@ namespace GameCult.Caching
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The one index rule, judged on the commit's result whatever the write order: within one document type and one
|
// The one index rule, a function of the state the admission would leave and never of staging, file or pull order:
|
||||||
// [CultIndex] alias, no two records hold one non-blank value. The holders of a value are the held key (unless this
|
// within one document type and one [CultIndex] alias, no two records hold one non-blank value. The holders of a
|
||||||
// admission replaces or evicts it) plus every landing key projecting it; two distinct holders refuse, naming both
|
// value are the held key (unless this admission replaces or evicts it) plus every landing key projecting it. Over
|
||||||
// keys and the index and never the value. One pass over what lands, one lookup per (index, value).
|
// every (type, alias, value) held at two or more distinct keys, the refusal names the least by (alias, its least
|
||||||
|
// key, its second-least key, schema name), each compared by UTF-16 code unit, and never names the value.
|
||||||
private void RefuseIndexSharing(VariantPlan plan, IReadOnlyList<CultStoredDocument> evicted)
|
private void RefuseIndexSharing(VariantPlan plan, IReadOnlyList<CultStoredDocument> evicted)
|
||||||
{
|
{
|
||||||
var landing = plan.Admitted.Concat(plan.Dependents).ToList();
|
var landing = plan.Admitted.Concat(plan.Dependents).ToList();
|
||||||
|
|
@ -3520,17 +3582,34 @@ namespace GameCult.Caching
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var ((index, value), landedKeys) in landed.OrderBy(pair => pair.Key.Index.Alias, StringComparer.Ordinal))
|
(string Alias, string First, string Second, string Schema)? least = null;
|
||||||
|
foreach (var ((index, value), landedKeys) in landed)
|
||||||
{
|
{
|
||||||
var holders = landedKeys.AsEnumerable();
|
var holders = landedKeys.AsEnumerable();
|
||||||
if (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKey) && !replaced.Contains(heldKey))
|
if (_indexes.TryGetValue(index, out var held) && held.TryGetValue(value, out var heldKey) && !replaced.Contains(heldKey))
|
||||||
holders = holders.Append(heldKey);
|
holders = holders.Append(heldKey);
|
||||||
var distinct = holders.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).Take(2).ToArray();
|
var distinct = holders.Distinct(StringComparer.Ordinal).OrderBy(key => key, StringComparer.Ordinal).Take(2).ToArray();
|
||||||
if (distinct.Length > 1)
|
if (distinct.Length < 2)
|
||||||
throw new InvalidOperationException(
|
continue;
|
||||||
$"Records {distinct[0]} and {distinct[1]} ({schemaNames[index.Type]}) " +
|
var candidate = (index.Alias, distinct[0], distinct[1], schemaNames[index.Type]);
|
||||||
$"both hold unique index '{index.Alias}'; an index value names one record.");
|
if (least == null || CompareRefusals(candidate, least.Value) < 0)
|
||||||
|
least = candidate;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (least is { } pair)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Records {pair.First} and {pair.Second} ({pair.Schema}) " +
|
||||||
|
$"both hold unique index '{pair.Alias}'; an index value names one record.");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int CompareRefusals(
|
||||||
|
(string Alias, string First, string Second, string Schema) left,
|
||||||
|
(string Alias, string First, string Second, string Schema) right)
|
||||||
|
{
|
||||||
|
var order = StringComparer.Ordinal.Compare(left.Alias, right.Alias);
|
||||||
|
if (order == 0) order = StringComparer.Ordinal.Compare(left.First, right.First);
|
||||||
|
if (order == 0) order = StringComparer.Ordinal.Compare(left.Second, right.Second);
|
||||||
|
return order != 0 ? order : StringComparer.Ordinal.Compare(left.Schema, right.Schema);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static InvalidOperationException CycleRefusal(IEnumerable<string> keys) =>
|
private static InvalidOperationException CycleRefusal(IEnumerable<string> keys) =>
|
||||||
|
|
@ -3919,11 +3998,21 @@ namespace GameCult.Caching
|
||||||
string version,
|
string version,
|
||||||
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
|
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
|
||||||
Func<Type, byte[], object> deserializePayload,
|
Func<Type, byte[], object> deserializePayload,
|
||||||
|
out CultSchemaMigrationReport report) =>
|
||||||
|
ToStoredDocument(record, version, catalog, (type, payload) => new CultDecoded(deserializePayload(type, payload), null), out report);
|
||||||
|
|
||||||
|
// The same, for a decode that notes the extra slots it read past (DeserializeMarked): the mark is turned into
|
||||||
|
// CultStoredDocument.UnreadSlots here, the one place that knows both the payload's marks and the catalog comparison.
|
||||||
|
internal CultStoredDocument ToStoredDocument(
|
||||||
|
CultPersistedRecord record,
|
||||||
|
string version,
|
||||||
|
IReadOnlyCollection<CultSchemaCatalogEntry> catalog,
|
||||||
|
Func<Type, byte[], CultDecoded> deserializePayload,
|
||||||
out CultSchemaMigrationReport report)
|
out CultSchemaMigrationReport report)
|
||||||
{
|
{
|
||||||
var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog);
|
var resolution = Registry.ResolvePersistedSchemaDetailed(record.SchemaId, catalog);
|
||||||
report = resolution.Report;
|
report = resolution.Report;
|
||||||
var readWhole = report.IgnoredExtraSlots.Length == 0;
|
CultUnreadSlots? unread = null;
|
||||||
if (record.Variant != null)
|
if (record.Variant != null)
|
||||||
{
|
{
|
||||||
var descriptor = resolution.Descriptor;
|
var descriptor = resolution.Descriptor;
|
||||||
|
|
@ -3937,13 +4026,22 @@ namespace GameCult.Caching
|
||||||
.ToArray();
|
.ToArray();
|
||||||
if (shed.Length > 0)
|
if (shed.Length > 0)
|
||||||
{
|
{
|
||||||
// A slot the schema diff already reported is not reported twice.
|
// A slot the schema diff already reported is not reported twice. One it did not report is a mark nothing explains:
|
||||||
|
// the variant is read-only and is never rewritten.
|
||||||
var unreported = shed.Except(report.IgnoredExtraSlots).ToArray();
|
var unreported = shed.Except(report.IgnoredExtraSlots).ToArray();
|
||||||
|
if (unreported.Length > 0)
|
||||||
|
unread = new CultUnreadSlots(
|
||||||
|
descriptor.DocumentType.Name,
|
||||||
|
unreported.Max() + 1,
|
||||||
|
descriptor.RichMembers.Select(member => member.Slot + 1).DefaultIfEmpty(0).Max(),
|
||||||
|
1,
|
||||||
|
allAtDepthZero: true,
|
||||||
|
explained: false);
|
||||||
report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(unreported).OrderBy(slot => slot).ToArray();
|
report.IgnoredExtraSlots = report.IgnoredExtraSlots.Concat(unreported).OrderBy(slot => slot).ToArray();
|
||||||
report.Warnings = report.Warnings.Concat(unreported.Select(slot => new CultSchemaMigrationWarning
|
report.Warnings = report.Warnings.Concat(unreported.Select(slot => new CultSchemaMigrationWarning
|
||||||
{
|
{
|
||||||
Code = "ignored_extra_slot",
|
Code = "ignored_extra_slot",
|
||||||
Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the override is ignored and shed on flush."
|
Message = $"Variant {record.Key} overrides slot {slot}, which {descriptor.DocumentType.Name} does not have; the variant is read-only."
|
||||||
})).ToArray();
|
})).ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3959,16 +4057,17 @@ namespace GameCult.Caching
|
||||||
null)
|
null)
|
||||||
{
|
{
|
||||||
StoredSchemaId = record.SchemaId,
|
StoredSchemaId = record.SchemaId,
|
||||||
StoredSchemaReadWhole = readWhole,
|
StoredSchemaReadWhole = report.IgnoredExtraSlots.Length == 0,
|
||||||
|
UnreadSlots = unread,
|
||||||
StoredPublishedBy = PublishedByIn(record.SchemaId, catalog),
|
StoredPublishedBy = PublishedByIn(record.SchemaId, catalog),
|
||||||
StoredVersion = version
|
StoredVersion = version
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
object document;
|
CultDecoded decoded;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
document = deserializePayload(resolution.Descriptor.DocumentType, record.Payload);
|
decoded = deserializePayload(resolution.Descriptor.DocumentType, record.Payload);
|
||||||
}
|
}
|
||||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
{
|
{
|
||||||
|
|
@ -3977,14 +4076,20 @@ namespace GameCult.Caching
|
||||||
for (var cause = exception; cause != null; cause = cause.InnerException) causes.Add(cause.Message);
|
for (var cause = exception; cause != null; cause = cause.InnerException) causes.Add(cause.Message);
|
||||||
throw new InvalidDataException($"Record '{record.Key}' (schema '{record.SchemaId}') cannot be decoded: {string.Join(" <- ", causes.Distinct())}", exception);
|
throw new InvalidDataException($"Record '{record.Key}' (schema '{record.SchemaId}') cannot be decoded: {string.Join(" <- ", causes.Distinct())}", exception);
|
||||||
}
|
}
|
||||||
|
// Explained only when every mark is at depth 0 and the schema comparison under this id reported every slot the mark carries
|
||||||
|
// past the reader: consent reaches no slot above the highest reported (none when nothing was reported).
|
||||||
|
unread = decoded.UnreadSlots is { } marked
|
||||||
|
? marked.Explaining(marked.AllAtDepthZero && marked.SlotCount <= report.IgnoredExtraSlots.DefaultIfEmpty(-1).Max() + 1)
|
||||||
|
: null;
|
||||||
return new CultStoredDocument(
|
return new CultStoredDocument(
|
||||||
new CultRecordKey(record.Key),
|
new CultRecordKey(record.Key),
|
||||||
record.StoredAt,
|
record.StoredAt,
|
||||||
resolution.Descriptor,
|
resolution.Descriptor,
|
||||||
document)
|
decoded.Document)
|
||||||
{
|
{
|
||||||
StoredSchemaId = record.SchemaId,
|
StoredSchemaId = record.SchemaId,
|
||||||
StoredSchemaReadWhole = readWhole,
|
StoredSchemaReadWhole = report.IgnoredExtraSlots.Length == 0 && unread == null,
|
||||||
|
UnreadSlots = unread,
|
||||||
StoredPublishedBy = PublishedByIn(record.SchemaId, catalog),
|
StoredPublishedBy = PublishedByIn(record.SchemaId, catalog),
|
||||||
StoredVersion = version
|
StoredVersion = version
|
||||||
};
|
};
|
||||||
|
|
@ -4023,22 +4128,37 @@ namespace GameCult.Caching
|
||||||
// catalog entry that published the held record's id when it was loaded, the question the store's judge asks of the file.
|
// catalog entry that published the held record's id when it was loaded, the question the store's judge asks of the file.
|
||||||
internal void RefuseUndeclared(string key, CultDocumentDescriptor? written)
|
internal void RefuseUndeclared(string key, CultDocumentDescriptor? written)
|
||||||
{
|
{
|
||||||
var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal));
|
if (Refusal(key, written) is { } refusal) throw refusal;
|
||||||
if (foreign != null)
|
|
||||||
throw Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) });
|
|
||||||
if (Entries.TryGetValue(key, out var held) && !HeldOwned(held, written))
|
|
||||||
throw Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The refusal of a write (of the written type) or a removal (written null) of the key, or null when this store's view lets it
|
||||||
|
// through. The one answer behind RefuseUndeclared and CultCache.IsWritable, so the query and the write cannot disagree.
|
||||||
|
internal CultSchemaConflictException? Refusal(string key, CultDocumentDescriptor? written)
|
||||||
|
{
|
||||||
|
var foreign = _foreignRecords.FirstOrDefault(record => string.Equals(record.Key, key, StringComparison.Ordinal));
|
||||||
|
if (foreign != null)
|
||||||
|
return Overwrites(null, new[] { (foreign.Key, foreign.SchemaId, foreign.SchemaName) });
|
||||||
|
if (!Entries.TryGetValue(key, out var held) || HeldOwned(held, written))
|
||||||
|
return null;
|
||||||
|
return held.UnreadSlots is { Explained: false } unread
|
||||||
|
? UnreadSlotsRefusal(held, key, unread)
|
||||||
|
: Overwrites(null, new[] { (key, held.StoredSchemaId, held.Descriptor.SchemaName) });
|
||||||
|
}
|
||||||
|
|
||||||
|
// The one ownership predicate of a held record. A mark nothing explains (anything but a depth-0 slot the catalog comparison
|
||||||
|
// reported ignored) keeps the record unowned whatever is declared: declaring the reader's own id consents to nothing.
|
||||||
private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) =>
|
private bool HeldOwned(CultStoredDocument held, CultDocumentDescriptor? written) =>
|
||||||
written == null
|
written == null
|
||||||
? Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole
|
? !Unexplained(held) && (Registry.Declared(held.StoredSchemaId) || held.StoredSchemaReadWhole)
|
||||||
: UpsertOwned(held, written);
|
: UpsertOwned(held, written);
|
||||||
|
|
||||||
|
private static bool Unexplained(CultStoredDocument held) => held.UnreadSlots is { Explained: false };
|
||||||
|
|
||||||
// Whether a write of the written type may replace the record a cache holds: WrittenOwns over the entry that published the held
|
// Whether a write of the written type may replace the record a cache holds: WrittenOwns over the entry that published the held
|
||||||
// record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed
|
// record's id when it was loaded (a record this cache wrote is published by its own type). The one answer for a store-backed
|
||||||
// cache's staging and a store-less cache's.
|
// cache's staging and a store-less cache's.
|
||||||
internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) =>
|
internal static bool UpsertOwned(CultStoredDocument held, CultDocumentDescriptor written) =>
|
||||||
|
!Unexplained(held) &&
|
||||||
CultDocumentRegistry.WrittenOwns(
|
CultDocumentRegistry.WrittenOwns(
|
||||||
written.ToCatalogEntry(),
|
written.ToCatalogEntry(),
|
||||||
held.StoredSchemaId,
|
held.StoredSchemaId,
|
||||||
|
|
@ -4340,6 +4460,16 @@ namespace GameCult.Caching
|
||||||
refused[0].Key,
|
refused[0].Key,
|
||||||
inner);
|
inner);
|
||||||
|
|
||||||
|
// The refusal of a write or removal of a record held read-only for the slots its payload carries past this build's. It names the
|
||||||
|
// type and the counts, never a slot value.
|
||||||
|
private CultSchemaConflictException UnreadSlotsRefusal(CultStoredDocument held, string key, CultUnreadSlots unread) => new(
|
||||||
|
$"{held.Descriptor.SchemaName} record {key} is read-only: {unread.TypeName} was read with {unread.SlotCount} slots where this build declares {unread.DeclaredCount} " +
|
||||||
|
$"({unread.Occurrences} object(s) in the record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or write it from that build.",
|
||||||
|
held.StoredSchemaId,
|
||||||
|
new[] { held.Descriptor.SchemaName },
|
||||||
|
key,
|
||||||
|
unread);
|
||||||
|
|
||||||
// A flush or commit succeeded: nothing staged is left, writes or checks.
|
// A flush or commit succeeded: nothing staged is left, writes or checks.
|
||||||
protected void MarkFlushSucceeded()
|
protected void MarkFlushSucceeded()
|
||||||
{
|
{
|
||||||
|
|
@ -4443,6 +4573,9 @@ namespace GameCult.Caching
|
||||||
protected abstract byte[] SerializePayload(object document, Type documentType);
|
protected abstract byte[] SerializePayload(object document, Type documentType);
|
||||||
protected abstract object DeserializePayload(Type documentType, byte[] payload);
|
protected abstract object DeserializePayload(Type documentType, byte[] payload);
|
||||||
|
|
||||||
|
// A decode that also notes the slots it read past. A store kind that does not override it never marks.
|
||||||
|
internal virtual CultDecoded DeserializeMarked(Type documentType, byte[] payload) => new(DeserializePayload(documentType, payload), null);
|
||||||
|
|
||||||
public override string ToString() => FileInfo.FullName;
|
public override string ToString() => FileInfo.FullName;
|
||||||
|
|
||||||
public override void PullAll() => Held(PullAllCore);
|
public override void PullAll() => Held(PullAllCore);
|
||||||
|
|
@ -4477,7 +4610,7 @@ namespace GameCult.Caching
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializePayload, out var report);
|
var stored = ToStoredDocument(record, version, snapshot.SchemaCatalog, DeserializeMarked, out var report);
|
||||||
reports.Add(report);
|
reports.Add(report);
|
||||||
persisted[stored.Key.Value] = stored;
|
persisted[stored.Key.Value] = stored;
|
||||||
}
|
}
|
||||||
|
|
@ -4945,7 +5078,7 @@ namespace GameCult.Caching
|
||||||
var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal));
|
var record = durable?.Snapshot.Records.FirstOrDefault(candidate => string.Equals(candidate.Key, key, StringComparison.Ordinal));
|
||||||
if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null)
|
if (record == null || Foreign(record, durable!.Snapshot.SchemaCatalog, durable.Version(key)) != null)
|
||||||
return null;
|
return null;
|
||||||
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializePayload, out _);
|
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializeMarked, out _);
|
||||||
}
|
}
|
||||||
|
|
||||||
// The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes:
|
// The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes:
|
||||||
|
|
|
||||||
|
|
@ -399,7 +399,7 @@ namespace GameCult.Caching
|
||||||
continue;
|
continue;
|
||||||
if (shape.Derive == null && !IsRandomId(id!))
|
if (shape.Derive == null && !IsRandomId(id!))
|
||||||
throw new CultElementIdException(
|
throw new CultElementIdException(
|
||||||
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex; for readable ids, derive them from a member with [CultElementId(nameof(Member))].",
|
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters. An id you write by hand must be 12 lowercase hex. To keep an existing readable id, leave that member as it is and add a trailing keyed string member marked [CultElementId(nameof(Member))], where Member is the existing one; the element id is derived from it once and then persisted.",
|
||||||
rootPath, path, id);
|
rootPath, path, id);
|
||||||
if (!taken.Add(id!))
|
if (!taken.Add(id!))
|
||||||
throw Duplicate(id!, path);
|
throw Duplicate(id!, path);
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,8 @@ namespace GameCult.Caching
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the
|
/// A write the catalog cannot describe: two schemas that share a schema id and disagree on the schema name (both arrived with the
|
||||||
/// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a
|
/// records, neither registered), a record whose schema id no chosen entry publishes, or a write that would replace or remove a
|
||||||
/// <see cref="CultForeignRecord"/>. Nothing is written, so the store stays as it was. It names the id, the schema names involved
|
/// <see cref="CultForeignRecord"/>, or a write or removal of a record held read-only because its payload carries slots past the
|
||||||
|
/// reader's declared slots (<see cref="UnreadSlots"/> is set). Nothing is written, so the store stays as it was. It names the id, the schema names involved
|
||||||
/// and a record key.
|
/// and a record key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CultSchemaConflictException : InvalidOperationException
|
public sealed class CultSchemaConflictException : InvalidOperationException
|
||||||
|
|
@ -27,8 +28,18 @@ namespace GameCult.Caching
|
||||||
RecordKey = recordKey;
|
RecordKey = recordKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>The refusal of a record held read-only for the slots it carries past this build's: the same refusal, with the finding.</summary>
|
||||||
|
public CultSchemaConflictException(string message, string schemaId, string[] schemaNames, string recordKey, CultUnreadSlots unreadSlots)
|
||||||
|
: this(message, schemaId, schemaNames, recordKey)
|
||||||
|
{
|
||||||
|
UnreadSlots = unreadSlots;
|
||||||
|
}
|
||||||
|
|
||||||
public string SchemaId { get; }
|
public string SchemaId { get; }
|
||||||
public string[] SchemaNames { get; }
|
public string[] SchemaNames { get; }
|
||||||
public string RecordKey { get; }
|
public string RecordKey { get; }
|
||||||
|
|
||||||
|
/// <summary>Set when the refusal is for unread slots; null for every other refusal.</summary>
|
||||||
|
public CultUnreadSlots? UnreadSlots { get; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
54
src/GameCult.Caching/CultUnreadSlots.cs
Normal file
54
src/GameCult.Caching/CultUnreadSlots.cs
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
namespace GameCult.Caching
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// What a store load found in a record's payload past the slots this build declares, at any depth: the first type read with extra
|
||||||
|
/// slots and how many slots it carried against how many it declares, and how many objects in the record did. It holds no slot
|
||||||
|
/// value, so it can be named in a refusal. A record carrying it is read-only (<see cref="CultCache.IsWritable"/>) unless every
|
||||||
|
/// mark is explained.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class CultUnreadSlots
|
||||||
|
{
|
||||||
|
internal CultUnreadSlots(string typeName, int slotCount, int declaredCount, int occurrences, bool allAtDepthZero, bool explained)
|
||||||
|
{
|
||||||
|
TypeName = typeName;
|
||||||
|
SlotCount = slotCount;
|
||||||
|
DeclaredCount = declaredCount;
|
||||||
|
Occurrences = occurrences;
|
||||||
|
AllAtDepthZero = allAtDepthZero;
|
||||||
|
Explained = explained;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The first type read with extra slots.</summary>
|
||||||
|
public string TypeName { get; }
|
||||||
|
|
||||||
|
/// <summary>The slots that object carried.</summary>
|
||||||
|
public int SlotCount { get; }
|
||||||
|
|
||||||
|
/// <summary>The slots this build declares for that type: the array its own formatter writes for one instance.</summary>
|
||||||
|
public int DeclaredCount { get; }
|
||||||
|
|
||||||
|
/// <summary>The objects in the record that carried extra slots.</summary>
|
||||||
|
public int Occurrences { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// True only when every mark is at depth 0 and the schema comparison under the id the record was stored under reported every
|
||||||
|
/// slot the mark carries past the reader (<see cref="SlotCount"/> is at most the highest reported slot plus one): the loss a
|
||||||
|
/// compatible-id declaration consents to (cultlib-gaps:ruling:slot-mark-consent). Any other mark leaves the record read-only
|
||||||
|
/// whatever is declared.
|
||||||
|
/// </summary>
|
||||||
|
public bool Explained { get; }
|
||||||
|
|
||||||
|
// Every mark the decode noted sat in the record's own object, the half of "explained" the payload decides.
|
||||||
|
internal bool AllAtDepthZero { get; }
|
||||||
|
|
||||||
|
internal CultUnreadSlots Explaining(bool explained) =>
|
||||||
|
new(TypeName, SlotCount, DeclaredCount, Occurrences, AllAtDepthZero, explained);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A payload decoded by a store kind, with the extra slots the decode noted. A store kind that does not note them reports none.
|
||||||
|
internal readonly struct CultDecoded(object document, CultUnreadSlots? unreadSlots)
|
||||||
|
{
|
||||||
|
public object Document { get; } = document;
|
||||||
|
public CultUnreadSlots? UnreadSlots { get; } = unreadSlots;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -228,6 +228,60 @@ the text of that member (a string or number on the same type, unique within its
|
||||||
list), and a derived id may be any text. A readable name set by hand on a
|
list), and a derived id may be any text. A readable name set by hand on a
|
||||||
non-derived list id is refused; the same member outside a list is unconstrained.
|
non-derived list id is refused; the same member outside a list is unconstrained.
|
||||||
|
|
||||||
|
### A type that already has a readable id
|
||||||
|
|
||||||
|
If the element type already has a readable string id (`Name`, `Slug`, a catalog
|
||||||
|
key), leave it as it is. Add a trailing keyed string member marked
|
||||||
|
`[CultElementId(nameof(ExistingId))]`; the element id is derived from the
|
||||||
|
existing member on load or first write, and then persisted:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class Recipe
|
||||||
|
{
|
||||||
|
[Key(0)] public string Name = string.Empty; // existing readable id
|
||||||
|
[Key(1)] public int Yield;
|
||||||
|
[Key(2)] [CultElementId(nameof(Name))] public string ElementId = string.Empty; // new, last
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Marking `Name` itself `[CultElementId]` is refused (`Name` is not 12 lowercase
|
||||||
|
hex), and that is deliberate:
|
||||||
|
|
||||||
|
- Minted and hand-written ids share one grammar, 12 lowercase hex, so they can
|
||||||
|
never collide; a derived id is the one readable form and is checked for
|
||||||
|
uniqueness within its list.
|
||||||
|
- An id the consumer can edit is not a stable override address. Variants address
|
||||||
|
an element by its id, so a field that is renamed in place would silently move
|
||||||
|
the overrides to a different element, or to none.
|
||||||
|
- A derived id is persisted once. Renaming `Name` later does not change
|
||||||
|
`ElementId`, so the element and every override that addresses it stay put.
|
||||||
|
`ElementId` is the address; `Name` goes back to being ordinary data.
|
||||||
|
|
||||||
|
Adopting the recipe on data that already exists: the existing member becomes an
|
||||||
|
id source, so it must be non-empty and unique within its list. A record with a
|
||||||
|
duplicate value and an empty one both refuse to load
|
||||||
|
(`CultElementIdException`), so clean such data first.
|
||||||
|
|
||||||
|
Take the next free `[Key]` slot, after the highest key on the type (and above
|
||||||
|
the subclasses' keys on a union base).
|
||||||
|
|
||||||
|
## Store format and rollback
|
||||||
|
|
||||||
|
A store that holds an element id is written as `cultcache.store.v3` (a directory
|
||||||
|
store as `cultcache.store.v5.directory-content-addressed-pages`, `v4` otherwise).
|
||||||
|
A single-file store that holds a variant but no id is `v2`, and a store with
|
||||||
|
neither stays `v1`. The mark follows content: a whole-store write that leaves no
|
||||||
|
element holding an id drops it.
|
||||||
|
|
||||||
|
Refusal by name depends on the store kind and the pin. A **directory** store (`v5`) is refused by every Unity pin from `cultlib-unity-v1.0.57` and `caching-unity-v1.2.0` on (the earliest tags of each line whose `DirectoryMessagePackBackingStore` reads only the `v4` stamp: "only ... is readable"). A **single-file** store (`v3`) is refused only from `cultlib-unity-v2.0.0` and `caching-unity-v2.0.0`: `cultlib-unity-v1.0.61` and `caching-unity-v1.4.0` read the header string and never compare it. The NuGet line (`cultlib-v1.0.32`, the last `cultlib-v` tag) refuses neither: its directory store writes `cultcache.store.v1.directory` and never checks the stamp on read. Rolling back a single-file store to any of those pins reads it silently, drops the ids, and can rewrite the elements without them. That is the dangerous case.
|
||||||
|
|
||||||
|
Rolling a consumer back after it has written a v3 or v5 store is therefore
|
||||||
|
one-way. The only carry-back is narrow and lossy: a whole-store write that leaves
|
||||||
|
no id anywhere in the store (every record rewritten without list elements) writes
|
||||||
|
the store unmarked again, and the ids are gone. Otherwise restore from a copy
|
||||||
|
taken before upgrading. Upgrade every reader before any writer.
|
||||||
|
|
||||||
## Persistence discipline
|
## Persistence discipline
|
||||||
|
|
||||||
Persistence stays manual by default. Mutations mark the cache and attached
|
Persistence stays manual by default. Mutations mark the cache and attached
|
||||||
|
|
|
||||||
|
|
@ -284,7 +284,10 @@ namespace GameCult.Mesh
|
||||||
throw new ArgumentException($"Expected provider command schema '{CultMeshProviderSessionWireContract.CommandSchema}'.", nameof(document));
|
throw new ArgumentException($"Expected provider command schema '{CultMeshProviderSessionWireContract.CommandSchema}'.", nameof(document));
|
||||||
if (!string.Equals(document.PayloadEncoding, "messagepack", StringComparison.Ordinal))
|
if (!string.Equals(document.PayloadEncoding, "messagepack", StringComparison.Ordinal))
|
||||||
throw new ArgumentException("Provider command document must use MessagePack.", nameof(document));
|
throw new ArgumentException("Provider command document must use MessagePack.", nameof(document));
|
||||||
var command = MessagePackSerializer.Deserialize<CultMeshProviderCommandWire>(document.Payload, Options);
|
// A raw record's payload is one MessagePack value (ruling trailing-bytes-refuse): a tail is refused by count, never walked.
|
||||||
|
var command = MessagePackSerializer.Deserialize<CultMeshProviderCommandWire>(new ReadOnlyMemory<byte>(document.Payload), Options, out var read);
|
||||||
|
if (read < document.Payload.Length)
|
||||||
|
throw new MessagePackSerializationException($"Provider command document: {document.Payload.Length - read} bytes after its first MessagePack value (refused on read).");
|
||||||
if (command == null) throw new MessagePackSerializationException("Provider command decoded to null.");
|
if (command == null) throw new MessagePackSerializationException("Provider command decoded to null.");
|
||||||
Validate(command);
|
Validate(command);
|
||||||
return command;
|
return command;
|
||||||
|
|
|
||||||
|
|
@ -641,23 +641,19 @@ namespace GameCult.Networking
|
||||||
string storedAt,
|
string storedAt,
|
||||||
CultNetDocumentMessageOptions? options = null)
|
CultNetDocumentMessageOptions? options = null)
|
||||||
{
|
{
|
||||||
var binding = GetByDocumentType(descriptor.DocumentType) ??
|
var binding = GetByDocumentType(descriptor.DocumentType);
|
||||||
new CultNetDocumentBinding(
|
|
||||||
descriptor.DocumentType,
|
|
||||||
descriptor.SchemaId,
|
|
||||||
value => CultDocumentMessagePackSerialization.SerializeUntyped(value, value.GetType(), _documents),
|
|
||||||
payload => CultDocumentMessagePackSerialization.DeserializeUntyped(descriptor.DocumentType, payload, _documents));
|
|
||||||
|
|
||||||
return new CultNetRawDocumentRecord
|
return new CultNetRawDocumentRecord
|
||||||
{
|
{
|
||||||
SchemaId = binding.SchemaId,
|
SchemaId = binding?.SchemaId ?? descriptor.SchemaId,
|
||||||
SchemaName = descriptor.SchemaName,
|
SchemaName = descriptor.SchemaName,
|
||||||
SchemaVersion = descriptor.SchemaVersion,
|
SchemaVersion = descriptor.SchemaVersion,
|
||||||
SchemaContentHash = descriptor.ContentHash,
|
SchemaContentHash = descriptor.ContentHash,
|
||||||
RecordKey = key.Value,
|
RecordKey = key.Value,
|
||||||
StoredAt = storedAt,
|
StoredAt = storedAt,
|
||||||
PayloadEncoding = "messagepack",
|
PayloadEncoding = "messagepack",
|
||||||
Payload = binding.PayloadSerializer(document),
|
Payload = binding?.PayloadSerializer(document) ??
|
||||||
|
CultDocumentMessagePackSerialization.SerializeUntyped(document, document.GetType(), _documents),
|
||||||
SourceRuntimeId = options?.SourceRuntimeId,
|
SourceRuntimeId = options?.SourceRuntimeId,
|
||||||
SourceAgentId = options?.SourceAgentId,
|
SourceAgentId = options?.SourceAgentId,
|
||||||
SourceRole = options?.SourceRole,
|
SourceRole = options?.SourceRole,
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,9 @@
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<IsPackable>false</IsPackable>
|
<IsPackable>false</IsPackable>
|
||||||
|
<!-- GameCult.Caching.MessagePack lets this check see CultUnionGuard.Inner (InternalsVisibleTo), which also shows it that
|
||||||
|
assembly's internal GeneratedMessagePackResolver; this project's own generated one is the type in use. -->
|
||||||
|
<NoWarn>$(NoWarn);CS0436</NoWarn>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<PropertyGroup Condition="'$(Aot)' == 'true'">
|
<PropertyGroup Condition="'$(Aot)' == 'true'">
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,8 @@ var documents = new (Type Type, object Value)[]
|
||||||
|
|
||||||
foreach (var (type, _) in documents)
|
foreach (var (type, _) in documents)
|
||||||
{
|
{
|
||||||
var formatter = options.Resolver.GetFormatterDynamic(type)!.GetType();
|
// The guard wraps a measured type's formatter; the generated formatter it delegates to is the one that runs.
|
||||||
|
var formatter = CultUnionGuard.Inner(options.Resolver.GetFormatterDynamic(type)!).GetType();
|
||||||
if (formatter.Assembly.IsDynamic || !(formatter.FullName ?? "").Contains("GeneratedMessagePackResolver"))
|
if (formatter.Assembly.IsDynamic || !(formatter.FullName ?? "").Contains("GeneratedMessagePackResolver"))
|
||||||
return Fail($"{type.Name} is formatted by {formatter.FullName}, not a generated formatter");
|
return Fail($"{type.Name} is formatted by {formatter.FullName}, not a generated formatter");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,8 @@ using MessagePack.Formatters;
|
||||||
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))]
|
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.OrderResolver))]
|
||||||
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))]
|
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.ListClaimResolver))]
|
||||||
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))]
|
[assembly: CultCacheFormatterResolver(typeof(UnionArmGuardTests.FbLaterClaimResolver))]
|
||||||
|
[assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.ResolvedBagResolver))]
|
||||||
|
[assembly: CultCacheFormatterResolver(typeof(SlotReadOnlyTests.RelayResolver))]
|
||||||
|
|
||||||
namespace GameCult.Caching.Tests
|
namespace GameCult.Caching.Tests
|
||||||
{
|
{
|
||||||
|
|
|
||||||
211
tests/GameCult.Caching.Tests/BytesAfterVectorTests.cs
Normal file
211
tests/GameCult.Caching.Tests/BytesAfterVectorTests.cs
Normal file
|
|
@ -0,0 +1,211 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
using MessagePack;
|
||||||
|
using NUnit.Framework;
|
||||||
|
using UnionSurface = GameCult.Caching.Tests.UnionParityVectorTests.UnionSurface;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.Tests
|
||||||
|
{
|
||||||
|
// A record's payload is exactly one MessagePack value. The vectors are union-surface.msgpack followed by a tail; the C#
|
||||||
|
// reference is their only writer (VectorFiles, CULTCACHE_WRITE_VECTORS=1) and every other runtime is held to bytes-after-outcomes.msgpack.
|
||||||
|
public class BytesAfterVectorTests
|
||||||
|
{
|
||||||
|
private const string Vector = "union-parity";
|
||||||
|
private const string Canary = "CANARY-bytes-after-7d1e";
|
||||||
|
private const string Clause = "bytes after its first MessagePack value";
|
||||||
|
private const int Limit = CultMessagePackSecurity.MaximumPayloadDepth;
|
||||||
|
|
||||||
|
private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(UnionSurface) });
|
||||||
|
|
||||||
|
private static byte[] Base => VectorFiles.Committed(Vector, "union-surface.msgpack");
|
||||||
|
|
||||||
|
private static byte[] DeepTail()
|
||||||
|
{
|
||||||
|
var text = System.Text.Encoding.ASCII.GetBytes(Canary);
|
||||||
|
return Enumerable.Repeat((byte)0x91, 300).Concat(new[] { (byte)(0xa0 | text.Length) }).Concat(text).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
// file, tail
|
||||||
|
private static (string File, byte[] Tail)[] Cases() => new (string, byte[])[]
|
||||||
|
{
|
||||||
|
("bytes-after-nil.msgpack", new byte[] { 0xc0 }),
|
||||||
|
("bytes-after-never-used.msgpack", new byte[] { 0xc1, 0xc1 }),
|
||||||
|
("bytes-after-deep-tail.msgpack", DeepTail()),
|
||||||
|
("bytes-after-second-document.msgpack", Base),
|
||||||
|
};
|
||||||
|
|
||||||
|
// The bounds row shape: [file, outcome, union, key|nil, detail].
|
||||||
|
private static byte[] OutcomesFile()
|
||||||
|
{
|
||||||
|
var buffer = new System.Buffers.ArrayBufferWriter<byte>();
|
||||||
|
var w = new MessagePackWriter(buffer);
|
||||||
|
var cases = Cases();
|
||||||
|
w.WriteArrayHeader(cases.Length);
|
||||||
|
foreach (var (file, _) in cases)
|
||||||
|
{
|
||||||
|
w.WriteArrayHeader(5);
|
||||||
|
w.Write(file);
|
||||||
|
w.Write("refused");
|
||||||
|
w.Write("UnionSurface");
|
||||||
|
w.WriteNil();
|
||||||
|
w.Write("bytes-after");
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Flush();
|
||||||
|
return buffer.WrittenSpan.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Message(Action action)
|
||||||
|
{
|
||||||
|
try { action(); }
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
var message = "";
|
||||||
|
for (var e = exception; e != null; e = e.InnerException) message += e.Message + "\n";
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new AssertionException("the payload was accepted");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void VectorsMatchAndRefuse()
|
||||||
|
{
|
||||||
|
var baseBytes = Base;
|
||||||
|
foreach (var (file, tail) in Cases()) VectorFiles.Check(Vector, file, baseBytes.Concat(tail).ToArray());
|
||||||
|
VectorFiles.Check(Vector, "bytes-after-outcomes.msgpack", OutcomesFile());
|
||||||
|
|
||||||
|
// From here only the committed files speak.
|
||||||
|
var reader = new MessagePackReader(VectorFiles.Committed(Vector, "bytes-after-outcomes.msgpack"));
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4));
|
||||||
|
for (var i = 0; i < 4; i++)
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(5));
|
||||||
|
var file = reader.ReadString()!;
|
||||||
|
Assert.That(reader.ReadString(), Is.EqualTo("refused"));
|
||||||
|
Assert.That(reader.ReadString(), Is.EqualTo("UnionSurface"));
|
||||||
|
Assert.That(reader.TryReadNil(), Is.True);
|
||||||
|
Assert.That(reader.ReadString(), Is.EqualTo("bytes-after"));
|
||||||
|
|
||||||
|
var payload = VectorFiles.Committed(Vector, file);
|
||||||
|
Assert.That(payload.Take(baseBytes.Length), Is.EqualTo(baseBytes), $"{file} begins with union-surface.msgpack");
|
||||||
|
var count = payload.Length - baseBytes.Length;
|
||||||
|
var message = Message(() => CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(UnionSurface), payload, Registry));
|
||||||
|
Assert.That(message, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"{count} {Clause}").And.Contain("refused on read"), file);
|
||||||
|
Assert.That(message, Does.Not.Contain(Canary).And.Not.Contain("the limit is"), file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The codec the store and the inspector read through is a path of its own: DecodeDocument<T> deserializes by the codec's
|
||||||
|
// delegate, so a tail it did not refuse would be read as a document.
|
||||||
|
[Test]
|
||||||
|
public void CodecDecodeDocumentRefusesATail()
|
||||||
|
{
|
||||||
|
var codec = CultCacheMessagePack.CreateCodec(Registry);
|
||||||
|
Assert.That(codec.DecodeDocument<UnionSurface>(new CultRecordKey("surface"), Base), Is.Not.Null);
|
||||||
|
var message = Message(() => codec.DecodeDocument<UnionSurface>(new CultRecordKey("surface"), Base.Concat(new byte[] { 0xc0 }).ToArray()));
|
||||||
|
Assert.That(message, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"1 {Clause}").And.Contain("refused on read"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void StoreLoadRefuses()
|
||||||
|
{
|
||||||
|
var work = Path.Combine(Path.GetTempPath(), $"cultlib-bytesafter-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(work);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(VectorFiles.Committed(Vector, "union-surface.cc"));
|
||||||
|
snapshot.Records[0].Payload = Base.Concat(new byte[] { 0xc0 }).ToArray();
|
||||||
|
var scratch = Path.Combine(work, "tail.cc");
|
||||||
|
File.WriteAllBytes(scratch, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
|
||||||
|
var load = Message(() => CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = Registry }).Dispose());
|
||||||
|
Assert.That(load, Does.Contain("surface").And.Contain($"1 {Clause}"));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(work, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A directory store names a page by the SHA-256 of its bytes and its manifest commits to that hash: the page holding the
|
||||||
|
// payload is rewritten with a tail, renamed, and the manifest told its new hash, so the store is well formed and only the payload is wrong.
|
||||||
|
[Test]
|
||||||
|
public async System.Threading.Tasks.Task DirectoryLoadPageRefuses()
|
||||||
|
{
|
||||||
|
var folder = Path.Combine(Path.GetTempPath(), $"cultlib-bytesafter-dir-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(folder);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.Combine(folder, "tail.cc");
|
||||||
|
var options = new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true, StoreFlushOnDispose = true };
|
||||||
|
using (var writer = CultCacheMessagePack.Create(path, options))
|
||||||
|
{
|
||||||
|
var value = CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(UnionSurface), Base, Registry);
|
||||||
|
await writer.UpsertAsync(typeof(UnionSurface), value, new CultRecordKey("surface"));
|
||||||
|
await writer.FlushAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
var rehashed = new System.Collections.Generic.List<(byte[] Old, byte[] New)>();
|
||||||
|
foreach (var file in Directory.EnumerateFiles(folder, "*.msgpack", SearchOption.AllDirectories).ToArray())
|
||||||
|
{
|
||||||
|
var bytes = File.ReadAllBytes(file);
|
||||||
|
if (bytes.AsSpan().IndexOf(Base) < 0) continue;
|
||||||
|
var page = CultDocumentMessagePackSerialization.DeserializePersistedRecord(bytes);
|
||||||
|
page.Payload = Base.Concat(new byte[] { 0xc0 }).ToArray();
|
||||||
|
var forged = CultDocumentMessagePackSerialization.SerializePersistedRecord(page);
|
||||||
|
File.Delete(file);
|
||||||
|
File.WriteAllBytes(Path.Combine(Path.GetDirectoryName(file)!, Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(forged)).ToLowerInvariant() + ".msgpack"), forged);
|
||||||
|
rehashed.Add((System.Security.Cryptography.SHA256.HashData(bytes), System.Security.Cryptography.SHA256.HashData(forged)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(rehashed, Is.Not.Empty, "the record page was found");
|
||||||
|
foreach (var file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Where(f => !f.EndsWith(".msgpack", StringComparison.Ordinal)).ToArray())
|
||||||
|
{
|
||||||
|
var bytes = File.ReadAllBytes(file);
|
||||||
|
foreach (var (old, @new) in rehashed)
|
||||||
|
for (var at = bytes.AsSpan().IndexOf(old); at >= 0; at = bytes.AsSpan().IndexOf(old)) @new.CopyTo(bytes, at);
|
||||||
|
File.WriteAllBytes(file, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
var thrown = Assert.CatchAsync(async () => { using var cache = await CultCacheMessagePack.OpenAsync(path, new CultCacheOpenOptions { Registry = Registry, UseDirectoryStore = true, ReadOnly = true }); });
|
||||||
|
var chain = "";
|
||||||
|
for (var e = thrown; e != null; e = e.InnerException) chain += e.Message + "\n";
|
||||||
|
Assert.That(chain, Does.Contain($"1 {Clause}"));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(folder, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class TwoValueFormatter : global::MessagePack.Formatters.IMessagePackFormatter<TwoValues>
|
||||||
|
{
|
||||||
|
public void Serialize(ref MessagePackWriter writer, TwoValues value, MessagePackSerializerOptions options)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(0);
|
||||||
|
writer.Write(7);
|
||||||
|
}
|
||||||
|
|
||||||
|
public TwoValues Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) { reader.Skip(); return new TwoValues(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
[CultDocument("tests.two_values", "tests.two_values.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
[MessagePackFormatter(typeof(TwoValueFormatter))]
|
||||||
|
public sealed class TwoValues
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Id = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void OwnFormatterWritingTwoValuesRefusedOnWrite()
|
||||||
|
{
|
||||||
|
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(TwoValues) });
|
||||||
|
var message = Message(() => CultDocumentMessagePackSerialization.SerializeUntyped(new TwoValues(), typeof(TwoValues), registry));
|
||||||
|
Assert.That(message, Does.Contain(typeof(TwoValues).FullName).And.Contain($"1 {Clause}").And.Contain("refused on write"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -142,7 +142,7 @@ namespace GameCult.Caching.Tests
|
||||||
public void TheHexRefusalNamesTheDerivedFormForReadableIds()
|
public void TheHexRefusalNamesTheDerivedFormForReadableIds()
|
||||||
{
|
{
|
||||||
var refusal = Assert.Throws<CultElementIdException>(() => Assign(new HHasIds { Items = { new EIded { Id = "root" } } }, "r", deterministic: false))!;
|
var refusal = Assert.Throws<CultElementIdException>(() => Assign(new HHasIds { Items = { new EIded { Id = "root" } } }, "r", deterministic: false))!;
|
||||||
Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]"));
|
Assert.That(refusal.Message, Does.Contain("12 lowercase hex").And.Contain("[CultElementId(nameof(Member))]").And.Contain("trailing keyed string member").And.Contain("existing readable id"));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
|
|
|
||||||
|
|
@ -389,6 +389,44 @@ namespace GameCult.Caching.Tests
|
||||||
Assert.That(refusal.Message, Does.Contain("'7'"));
|
Assert.That(refusal.Message, Does.Contain("'7'"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The documented recipe for a type that already has a readable id (src/GameCult.Caching/README.md, Element ids).
|
||||||
|
[Test]
|
||||||
|
public void ATrailingMemberDerivedFromAnExistingReadableIdIsPersistedOnceAndTheStoreIsMarkedV3()
|
||||||
|
{
|
||||||
|
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RecipeBook) });
|
||||||
|
var path = PathOf("recipes.cc");
|
||||||
|
var key = new CultRecordKey("book");
|
||||||
|
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }))
|
||||||
|
cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook { Recipes = { new IdRecipe { Name = "bread" }, new IdRecipe { Name = "soup" } } }, key));
|
||||||
|
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"));
|
||||||
|
|
||||||
|
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry }))
|
||||||
|
{
|
||||||
|
var book = cache.Get<RecipeBook>(key)!;
|
||||||
|
Assert.That(book.Recipes.Select(recipe => recipe.ElementId), Is.EqualTo(new[] { "bread", "soup" }));
|
||||||
|
book.Recipes[0].Name = "loaf";
|
||||||
|
cache.Commit(batch => batch.Upsert(typeof(RecipeBook), book, key));
|
||||||
|
}
|
||||||
|
|
||||||
|
using var again = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
|
||||||
|
var reread = again.Get<RecipeBook>(key)!;
|
||||||
|
Assert.That(reread.Recipes.Select(recipe => recipe.Name), Is.EqualTo(new[] { "loaf", "soup" }));
|
||||||
|
Assert.That(reread.Recipes.Select(recipe => recipe.ElementId), Is.EqualTo(new[] { "bread", "soup" }), "renaming the source does not move the element");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AWholeStoreWriteThatLeavesNoIdInTheStoreDropsTheMarkBackToV1()
|
||||||
|
{
|
||||||
|
var registry = CultDocumentRegistry.ForTypes(new[] { typeof(RecipeBook) });
|
||||||
|
var path = PathOf("drop.cc");
|
||||||
|
var key = new CultRecordKey("book");
|
||||||
|
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
|
||||||
|
cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook { Recipes = { new IdRecipe { Name = "bread" } } }, key));
|
||||||
|
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"));
|
||||||
|
cache.Commit(batch => batch.Upsert(typeof(RecipeBook), new RecipeBook(), key));
|
||||||
|
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the mark follows content: no element holds an id, so the store is unmarked");
|
||||||
|
}
|
||||||
|
|
||||||
// ---- C2a fix batch: the marker, refusals at load, override values, atomic minting ----
|
// ---- C2a fix batch: the marker, refusals at load, override values, atomic minting ----
|
||||||
|
|
||||||
private const string HexA = "aaaaaaaaaaaa";
|
private const string HexA = "aaaaaaaaaaaa";
|
||||||
|
|
@ -1359,6 +1397,20 @@ namespace GameCult.Caching.Tests
|
||||||
[Key(1)] public List<IdLabel> Labels { get; set; } = new();
|
[Key(1)] public List<IdLabel> Labels { get; set; } = new();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[CultDocument("tests.element_id_recipe_book", "tests.element_id_recipe_book.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class RecipeBook
|
||||||
|
{
|
||||||
|
[Key(0)] public List<IdRecipe> Recipes { get; set; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class IdRecipe
|
||||||
|
{
|
||||||
|
[Key(0)] public string Name { get; set; } = "";
|
||||||
|
[Key(1)] [CultElementId(nameof(Name))] public string ElementId { get; set; } = "";
|
||||||
|
}
|
||||||
|
|
||||||
[MessagePackObject]
|
[MessagePackObject]
|
||||||
public sealed class IdLabel
|
public sealed class IdLabel
|
||||||
{
|
{
|
||||||
|
|
|
||||||
371
tests/GameCult.Caching.Tests/KeyIdentityVectorTests.cs
Normal file
371
tests/GameCult.Caching.Tests/KeyIdentityVectorTests.cs
Normal file
|
|
@ -0,0 +1,371 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.Buffers;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using GameCult.Caching;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
using MessagePack;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.Tests
|
||||||
|
{
|
||||||
|
// tests/vectors/key-identity: a cache holds one record per key whatever its type, and a store holding a record no registered
|
||||||
|
// type claims keeps it (variants:ruling:cache-refuses-retype, write-ownership-by-written-type, unheld-removal-refused-at-commit).
|
||||||
|
// The C# reference is the only writer: under CULTCACHE_WRITE_VECTORS=1 the rows are run and what the reference answers is
|
||||||
|
// written; every other run walks the committed rows against the reference and compares the files byte for byte.
|
||||||
|
public class KeyIdentityVectorTests
|
||||||
|
{
|
||||||
|
private const string Vector = "key-identity";
|
||||||
|
|
||||||
|
private static readonly Type Alpha = Kind("KiAlpha", "tests.ki_alpha");
|
||||||
|
private static readonly Type Beta = Kind("KiBeta", "tests.ki_beta");
|
||||||
|
private static readonly Type Gamma = Kind("KiGamma", "tests.ki_gamma");
|
||||||
|
|
||||||
|
private static Type Kind(string type, string schema) =>
|
||||||
|
EmittedDocumentTypes.Emit(type, schema, schema + ".v1", new[] { new EmittedDocumentTypes.Field("Text", typeof(string), 0) });
|
||||||
|
|
||||||
|
private static readonly Dictionary<string, Type> Types = new() { ["alpha"] = Alpha, ["beta"] = Beta, ["gamma"] = Gamma };
|
||||||
|
|
||||||
|
private sealed record Op(string Kind, string? File = null, string[]? Types = null, string? Type = null, string? Key = null, string? Text = null);
|
||||||
|
|
||||||
|
private sealed record Row(string Label, string[] Registered, Op[] Ops);
|
||||||
|
|
||||||
|
// [refusal key or nil, held [type, key, text], file [key, schema name] or nil, foreign keys, kept foreign keys], all sorted.
|
||||||
|
private sealed record Expect(string? Refusal, string[][] Held, string[][]? File, string[] Foreign, string[] Kept);
|
||||||
|
|
||||||
|
private static Op Open(string? file) => new("open", file);
|
||||||
|
private static Op Attach(string file, params string[] home) => new("attach", file, home);
|
||||||
|
private static Op Reopen() => new("reopen");
|
||||||
|
private static Op Put(string type, string key, string text) => new("put", Type: type, Key: key, Text: text);
|
||||||
|
private static Op Delete(string type, string key) => new("delete", Type: type, Key: key);
|
||||||
|
|
||||||
|
private static readonly string[] AB = { "alpha", "beta" };
|
||||||
|
private static readonly string[] A = { "alpha" };
|
||||||
|
|
||||||
|
// Only the ops are typed here; every verdict is what the reference answers.
|
||||||
|
private static readonly Row[] Table =
|
||||||
|
{
|
||||||
|
new("retype-refused-held-in-file", AB, new[] { Open(null), Put("alpha", "k", "a"), Put("beta", "k", "b") }),
|
||||||
|
new("retype-by-remove-then-add", AB, new[] { Open(null), Put("alpha", "k", "a"), Delete("alpha", "k"), Put("beta", "k", "b") }),
|
||||||
|
new("retype-refused-memory-cache", AB, new[] { Open("memory"), Put("alpha", "k", "a"), Put("beta", "k", "b") }),
|
||||||
|
new("same-type-replaces", A, new[] { Open(null), Put("alpha", "k", "a"), Put("alpha", "k", "b") }),
|
||||||
|
new("retype-refused-after-reload", AB, new[] { Open(null), Put("alpha", "k", "a"), Reopen(), Put("beta", "k", "b") }),
|
||||||
|
new("foreign-loads", A, new[] { Open("foreign-store.cc") }),
|
||||||
|
new("foreign-kept-by-a-write", A, new[] { Open("foreign-store.cc"), Put("alpha", "m", "m") }),
|
||||||
|
new("foreign-key-write-refused", A, new[] { Open("foreign-store.cc"), Put("alpha", "k", "x") }),
|
||||||
|
new("foreign-key-delete-refused", A, new[] { Open("foreign-store.cc"), Delete("alpha", "k") }),
|
||||||
|
new("foreign-claimed-by-a-later-registration", new[] { "alpha", "gamma" }, new[] { Open("foreign-store.cc") }),
|
||||||
|
new("key-in-two-stores-refused", AB, new[] { Open("memory"), Attach("catalog-store.cc", "alpha"), Attach("run-store.cc", "beta") }),
|
||||||
|
new("same-type-replaces-memory-cache", A, new[] { Open("memory"), Put("alpha", "k", "a"), Put("alpha", "k", "b") }),
|
||||||
|
};
|
||||||
|
|
||||||
|
// The stores C# writes: [file name, registered types, (type, key, text) records]. Never typed by a runtime.
|
||||||
|
private static readonly (string File, string[] Registered, (string Type, string Key, string Text)[] Records)[] Stores =
|
||||||
|
{
|
||||||
|
("foreign-store.cc", new[] { "alpha", "gamma" }, new[] { ("alpha", "j", "a"), ("gamma", "k", "g") }),
|
||||||
|
("catalog-store.cc", new[] { "alpha" }, new[] { ("alpha", "qx", "a") }),
|
||||||
|
("run-store.cc", new[] { "beta" }, new[] { ("beta", "qx", "b") }),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static CultDocumentRegistry RegistryOf(IEnumerable<string> names) => CultDocumentRegistry.ForTypes(names.Select(name => Types[name]).ToArray());
|
||||||
|
|
||||||
|
private static object Doc(string type, string text) => EmittedDocumentTypes.New(Types[type], ("Text", text));
|
||||||
|
|
||||||
|
private static byte[] StoreBytes(string[] registered, (string Type, string Key, string Text)[] records)
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-kiv-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.Combine(directory, "store.cc");
|
||||||
|
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = RegistryOf(registered), StoreFlushOnDispose = true }))
|
||||||
|
{
|
||||||
|
cache.Clock = () => VectorFiles.StoredAt;
|
||||||
|
foreach (var record in records)
|
||||||
|
cache.UpsertAsync(Types[record.Type], Doc(record.Type, record.Text), new CultRecordKey(record.Key)).GetAwaiter().GetResult();
|
||||||
|
cache.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
return File.ReadAllBytes(path);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CultPersistedStoreSnapshot Snapshot(byte[] bytes) => CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes);
|
||||||
|
|
||||||
|
private static string Name(CultPersistedStoreSnapshot snapshot, CultPersistedRecord record) =>
|
||||||
|
snapshot.SchemaCatalog.Single(entry => entry.SchemaId == record.SchemaId).SchemaName;
|
||||||
|
|
||||||
|
private static string Short(string schemaName) => Types.Single(pair => pair.Key == schemaName.Replace("tests.ki_", "")).Key;
|
||||||
|
|
||||||
|
// The key as a delimited token: not a piece of a longer word, schema name or path.
|
||||||
|
private static bool Delimited(string message, string key) => Regex.IsMatch(message, $@"(?<![A-Za-z0-9_]){Regex.Escape(key)}(?![A-Za-z0-9_])");
|
||||||
|
|
||||||
|
private static string RecordBytes(CultPersistedRecord record) => $"{record.SchemaId}|{record.StoredAt}|{Convert.ToBase64String(record.Payload)}";
|
||||||
|
|
||||||
|
// Runs the ops and returns what the reference answers. Every op but the last must succeed; the last may refuse.
|
||||||
|
private static Expect Observe(Row row)
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-kiw-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
var registry = RegistryOf(row.Registered);
|
||||||
|
string? path = null;
|
||||||
|
byte[]? before = null;
|
||||||
|
CultCache? cache = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Exception? refusal = null;
|
||||||
|
string? refusedKey = null;
|
||||||
|
for (var i = 0; i < row.Ops.Length; i++)
|
||||||
|
{
|
||||||
|
var op = row.Ops[i];
|
||||||
|
try
|
||||||
|
{
|
||||||
|
switch (op.Kind)
|
||||||
|
{
|
||||||
|
case "open" or "reopen":
|
||||||
|
cache?.Dispose();
|
||||||
|
if (op.Kind == "open" && op.File == "memory")
|
||||||
|
{
|
||||||
|
cache = new CultCache(registry, CultCacheMessagePack.CreateCodec(registry));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (op.Kind == "open")
|
||||||
|
{
|
||||||
|
path = Path.Combine(directory, "store.cc");
|
||||||
|
if (op.File != null) File.WriteAllBytes(path, VectorFiles.Committed(Vector, op.File));
|
||||||
|
before = File.Exists(path) ? File.ReadAllBytes(path) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
cache = CultCacheMessagePack.Create(path!, new CultCacheOpenOptions { Registry = registry, StoreFlushOnDispose = true });
|
||||||
|
break;
|
||||||
|
case "attach":
|
||||||
|
{
|
||||||
|
var attached = Path.Combine(directory, op.File!);
|
||||||
|
File.WriteAllBytes(attached, VectorFiles.Committed(Vector, op.File!));
|
||||||
|
var held = cache!.AllStoredDocuments.Select(stored => stored.Key.Value).ToHashSet(StringComparer.Ordinal);
|
||||||
|
var shared = Snapshot(File.ReadAllBytes(attached)).Records.Select(record => record.Key).Where(held.Contains).ToArray();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
cache.AddBackingStore(new SingleFileMessagePackBackingStore(attached), op.Types!.Select(name => Types[name]).ToArray());
|
||||||
|
cache.PullAllBackingStoresAsync().GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
catch (InvalidOperationException) when (shared.Length == 1)
|
||||||
|
{
|
||||||
|
Assert.That(File.ReadAllBytes(attached), Is.EqualTo(VectorFiles.Committed(Vector, op.File!)), "a refused attach leaves the store as it was");
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case "put":
|
||||||
|
cache!.Commit(batch => batch.Upsert(Types[op.Type!], Doc(op.Type!, op.Text!), new CultRecordKey(op.Key!)));
|
||||||
|
break;
|
||||||
|
case "delete":
|
||||||
|
cache!.Commit(batch => batch.Remove(new CultRecordKey(op.Key!)));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new InvalidOperationException(op.Kind);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception failure) when (i == row.Ops.Length - 1 &&
|
||||||
|
(failure is CultSchemaConflictException || (op.Kind == "attach" && failure is InvalidOperationException)))
|
||||||
|
{
|
||||||
|
refusal = failure;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (refusal is CultSchemaConflictException conflict)
|
||||||
|
{
|
||||||
|
refusedKey = conflict.RecordKey;
|
||||||
|
Assert.That(row.Ops[^1].Key, Is.EqualTo(conflict.RecordKey), $"{row.Label}: the refusal names the key written or removed");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (refusal is InvalidOperationException && refusal is not CultSchemaConflictException)
|
||||||
|
{
|
||||||
|
// The key is read out of the refusal's own text, not taken from the row: the second-store refusal opens "Record <key> is held by".
|
||||||
|
var named = Regex.Match(refusal.Message, @"^Record (\S+) is held by");
|
||||||
|
Assert.That(named.Success, Is.True, $"{row.Label}: the refusal opens by naming a key: {refusal.Message}");
|
||||||
|
refusedKey = named.Groups[1].Value;
|
||||||
|
var shared = Snapshot(VectorFiles.Committed(Vector, "run-store.cc")).Records.Select(record => record.Key).ToArray();
|
||||||
|
Assert.That(shared, Does.Contain(refusedKey), $"{row.Label}: the key the refusal names is the shared one");
|
||||||
|
Assert.That(Regex.Matches(refusal.Message, Regex.Escape(refusedKey)).Count, Is.EqualTo(Regex.Matches(refusal.Message, $@"(?<![A-Za-z0-9_]){Regex.Escape(refusedKey)}(?![A-Za-z0-9_])").Count),
|
||||||
|
$"{row.Label}: the key occurs in the refusal only as a delimited token, never inside a schema name or other text");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (refusal != null)
|
||||||
|
Assert.That(Delimited(refusal.Message, refusedKey ?? "\0"), Is.True, $"{row.Label}: the refusal names the key as a token");
|
||||||
|
|
||||||
|
var heldNow = cache!.AllStoredDocuments
|
||||||
|
.Select(stored => new[] { Short(stored.Descriptor.SchemaName), stored.Key.Value, (string)EmittedDocumentTypes.Read(stored.Document, "Text")! })
|
||||||
|
.OrderBy(entry => entry[0], StringComparer.Ordinal).ThenBy(entry => entry[1], StringComparer.Ordinal).ToArray();
|
||||||
|
var foreign = cache.BackingStores.SelectMany(store => store.ForeignRecords).Select(record => record.Key).OrderBy(key => key, StringComparer.Ordinal).ToArray();
|
||||||
|
|
||||||
|
string[][]? file = null;
|
||||||
|
var kept = Array.Empty<string>();
|
||||||
|
if (path != null)
|
||||||
|
{
|
||||||
|
cache.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
var after = Snapshot(File.ReadAllBytes(path));
|
||||||
|
file = after.Records.OrderBy(record => record.Key, StringComparer.Ordinal).Select(record => new[] { record.Key, Name(after, record) }).ToArray();
|
||||||
|
var start = before == null ? null : Snapshot(before);
|
||||||
|
kept = foreign.Where(key =>
|
||||||
|
{
|
||||||
|
var original = start?.Records.SingleOrDefault(record => record.Key == key);
|
||||||
|
return original != null && RecordBytes(original) == RecordBytes(after.Records.Single(record => record.Key == key));
|
||||||
|
}).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Expect(refusedKey, heldNow, file, foreign, kept);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
cache?.Dispose();
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- rows.msgpack: [label, registered, ops, expect] ----
|
||||||
|
|
||||||
|
private static void WriteStrings(ref MessagePackWriter writer, IEnumerable<string?> values)
|
||||||
|
{
|
||||||
|
var list = values.ToArray();
|
||||||
|
writer.WriteArrayHeader(list.Length);
|
||||||
|
foreach (var value in list)
|
||||||
|
if (value == null) writer.WriteNil(); else writer.Write(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void WriteRows(ref MessagePackWriter writer, Row row, Expect expect)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(4);
|
||||||
|
writer.Write(row.Label);
|
||||||
|
WriteStrings(ref writer, row.Registered);
|
||||||
|
writer.WriteArrayHeader(row.Ops.Length);
|
||||||
|
foreach (var op in row.Ops)
|
||||||
|
{
|
||||||
|
var fields = op.Kind switch
|
||||||
|
{
|
||||||
|
"open" => new object?[] { op.Kind, op.File },
|
||||||
|
"attach" => new object?[] { op.Kind, op.File, op.Types },
|
||||||
|
"reopen" => new object?[] { op.Kind },
|
||||||
|
"put" => new object?[] { op.Kind, op.Type, op.Key, op.Text },
|
||||||
|
_ => new object?[] { op.Kind, op.Type, op.Key },
|
||||||
|
};
|
||||||
|
writer.WriteArrayHeader(fields.Length);
|
||||||
|
foreach (var field in fields)
|
||||||
|
switch (field)
|
||||||
|
{
|
||||||
|
case null: writer.WriteNil(); break;
|
||||||
|
case string[] list: WriteStrings(ref writer, list); break;
|
||||||
|
default: writer.Write((string)field); break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.WriteArrayHeader(5);
|
||||||
|
if (expect.Refusal == null) writer.WriteNil(); else WriteStrings(ref writer, new[] { expect.Refusal });
|
||||||
|
writer.WriteArrayHeader(expect.Held.Length);
|
||||||
|
foreach (var held in expect.Held) WriteStrings(ref writer, held);
|
||||||
|
if (expect.File == null) writer.WriteNil();
|
||||||
|
else
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(expect.File.Length);
|
||||||
|
foreach (var record in expect.File) WriteStrings(ref writer, record);
|
||||||
|
}
|
||||||
|
|
||||||
|
WriteStrings(ref writer, expect.Foreign);
|
||||||
|
WriteStrings(ref writer, expect.Kept);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] RowsFile()
|
||||||
|
{
|
||||||
|
var buffer = new ArrayBufferWriter<byte>();
|
||||||
|
var writer = new MessagePackWriter(buffer);
|
||||||
|
writer.WriteArrayHeader(Table.Length);
|
||||||
|
foreach (var row in Table)
|
||||||
|
WriteRows(ref writer, row, Observe(row));
|
||||||
|
writer.Flush();
|
||||||
|
return buffer.WrittenSpan.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string[] ReadStrings(ref MessagePackReader reader)
|
||||||
|
{
|
||||||
|
var values = new string[reader.ReadArrayHeader()];
|
||||||
|
for (var i = 0; i < values.Length; i++) values[i] = reader.ReadString()!;
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static (Row Row, Expect Expect) ReadRow(ref MessagePackReader reader)
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4));
|
||||||
|
var label = reader.ReadString()!;
|
||||||
|
var registered = ReadStrings(ref reader);
|
||||||
|
var ops = new Op[reader.ReadArrayHeader()];
|
||||||
|
for (var i = 0; i < ops.Length; i++)
|
||||||
|
{
|
||||||
|
var length = reader.ReadArrayHeader();
|
||||||
|
var kind = reader.ReadString()!;
|
||||||
|
ops[i] = kind switch
|
||||||
|
{
|
||||||
|
"open" => new Op(kind, reader.TryReadNil() ? null : reader.ReadString()),
|
||||||
|
"attach" => new Op(kind, reader.ReadString(), ReadStrings(ref reader)),
|
||||||
|
"reopen" => new Op(kind),
|
||||||
|
"put" => new Op(kind, Type: reader.ReadString(), Key: reader.ReadString(), Text: reader.ReadString()),
|
||||||
|
"delete" => new Op(kind, Type: reader.ReadString(), Key: reader.ReadString()),
|
||||||
|
_ => throw new InvalidOperationException(kind),
|
||||||
|
};
|
||||||
|
Assert.That(length, Is.EqualTo(kind switch { "reopen" => 1, "open" => 2, "delete" => 3, "attach" => 3, _ => 4 }), $"{label}: op {kind}");
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(5));
|
||||||
|
var refusal = reader.TryReadNil() ? null : ReadStrings(ref reader).Single();
|
||||||
|
var held = new string[reader.ReadArrayHeader()][];
|
||||||
|
for (var i = 0; i < held.Length; i++) held[i] = ReadStrings(ref reader);
|
||||||
|
string[][]? file = null;
|
||||||
|
if (!reader.TryReadNil())
|
||||||
|
{
|
||||||
|
file = new string[reader.ReadArrayHeader()][];
|
||||||
|
for (var i = 0; i < file.Length; i++) file[i] = ReadStrings(ref reader);
|
||||||
|
}
|
||||||
|
|
||||||
|
var foreign = ReadStrings(ref reader);
|
||||||
|
var kept = ReadStrings(ref reader);
|
||||||
|
return (new Row(label, registered, ops), new Expect(refusal, held, file, foreign, kept));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Render(Expect expect) =>
|
||||||
|
$"refusal={expect.Refusal ?? "-"} held={string.Join(";", expect.Held.Select(h => string.Join(",", h)))} " +
|
||||||
|
$"file={(expect.File == null ? "-" : string.Join(";", expect.File.Select(f => string.Join(",", f))))} " +
|
||||||
|
$"foreign={string.Join(",", expect.Foreign)} kept={string.Join(",", expect.Kept)}";
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void KeyIdentityRowsHoldTheirVerdicts()
|
||||||
|
{
|
||||||
|
foreach (var store in Stores)
|
||||||
|
VectorFiles.Check(Vector, store.File, StoreBytes(store.Registered, store.Records));
|
||||||
|
VectorFiles.Check(Vector, "rows.msgpack", RowsFile());
|
||||||
|
|
||||||
|
var reader = new MessagePackReader(VectorFiles.Committed(Vector, "rows.msgpack"));
|
||||||
|
var count = reader.ReadArrayHeader();
|
||||||
|
Assert.That(count, Is.EqualTo(Table.Length));
|
||||||
|
var walked = 0;
|
||||||
|
var failed = new List<string>();
|
||||||
|
for (var i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
var (row, expect) = ReadRow(ref reader);
|
||||||
|
var observed = Observe(row);
|
||||||
|
if (Render(observed) == Render(expect)) walked++;
|
||||||
|
else failed.Add($"{row.Label}: committed {Render(expect)} but the reference answers {Render(observed)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(failed, Is.Empty);
|
||||||
|
Assert.That(walked, Is.EqualTo(12), "rows walked");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -148,10 +148,25 @@ namespace GameCult.Caching.Tests
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void OnlyTheFirstValueIsThePayload()
|
public void OnlyOneValueIsAPayload()
|
||||||
{
|
{
|
||||||
Accepts(Cat(Bytes(0x90), Chain(Limit + 10)), "bytes after the first value are not walked");
|
var tails = new[]
|
||||||
Accepts(Cat(Bytes(0x01), Chain(Limit + 10)), "nor after a scalar");
|
{
|
||||||
|
(Cat(Bytes(0x90), Chain(Limit + 10)), Chain(Limit + 10).Length),
|
||||||
|
(Cat(Bytes(0x01), Chain(Limit + 10)), Chain(Limit + 10).Length),
|
||||||
|
(Bytes(0x90, 0xc1, 0xc1), 2),
|
||||||
|
};
|
||||||
|
foreach (var (payload, count) in tails)
|
||||||
|
foreach (var direction in new[] { PayloadDirection.Read, PayloadDirection.Write })
|
||||||
|
{
|
||||||
|
var message = Assert.Throws<MessagePackSerializationException>(() => CultPayloadDepth.Check(payload, typeof(PayloadDepthTests), direction))!.Message;
|
||||||
|
var word = direction == PayloadDirection.Read ? "read" : "write";
|
||||||
|
Assert.That(message, Is.EqualTo($"{typeof(PayloadDepthTests).FullName} payload has {count} bytes after its first MessagePack value (refused on {word})."));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tail is not walked: a tail nested past the limit is refused as bytes after the value, not as depth.
|
||||||
|
var deep = Assert.Throws<MessagePackSerializationException>(() => CultPayloadDepth.Check(Cat(Bytes(0x90), Chain(Limit + 10)), typeof(object), PayloadDirection.Read))!;
|
||||||
|
Assert.That(deep.Message, Does.Not.Contain("the limit is"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
639
tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs
Normal file
639
tests/GameCult.Caching.Tests/SlotReadOnlyTests.cs
Normal file
|
|
@ -0,0 +1,639 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.Buffers;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
using MessagePack;
|
||||||
|
using MessagePack.Formatters;
|
||||||
|
using NUnit.Framework;
|
||||||
|
using Surface = GameCult.Caching.Tests.UnionParityVectorTests.UnionSurface;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.Tests
|
||||||
|
{
|
||||||
|
// cultlib-gaps:ruling:extra-trailing-slots and :slot-mark-consent. A record whose payload carries a slot past its reader's declared
|
||||||
|
// slots, at any depth, is held read-only: every staged write or removal of it is refused before anything is staged. A compatible-id
|
||||||
|
// declaration consents only to depth-0 slots the catalog comparison reports ignored under a listed compatible id.
|
||||||
|
public class SlotReadOnlyTests
|
||||||
|
{
|
||||||
|
private const string Extra = "extra-slot";
|
||||||
|
|
||||||
|
// Slot 1 and 2 are retired gaps.
|
||||||
|
[CultDocument("tests.slot_gap", "tests.slot_gap.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class GapDoc
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Name = string.Empty;
|
||||||
|
[Key(3)] public string Tail = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly CultDocumentRegistry SurfaceRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(Surface) });
|
||||||
|
private static readonly CultDocumentRegistry GapRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(GapDoc) });
|
||||||
|
private static readonly CultRecordKey Marked = new("marked");
|
||||||
|
private static readonly CultRecordKey Clean = new("clean");
|
||||||
|
private static readonly CultRecordKey Unheld = new("unheld");
|
||||||
|
|
||||||
|
private string _directory = string.Empty;
|
||||||
|
|
||||||
|
[SetUp]
|
||||||
|
public void CreateDirectory()
|
||||||
|
{
|
||||||
|
_directory = Path.Combine(Path.GetTempPath(), $"cultlib-slotro-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_directory);
|
||||||
|
}
|
||||||
|
|
||||||
|
[TearDown]
|
||||||
|
public void DeleteDirectory()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(_directory)) Directory.Delete(_directory, recursive: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CultCache Open(string path, CultDocumentRegistry registry) =>
|
||||||
|
CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = registry });
|
||||||
|
|
||||||
|
private static byte[] Emit(WriterAction action)
|
||||||
|
{
|
||||||
|
var buffer = new ArrayBufferWriter<byte>();
|
||||||
|
var writer = new MessagePackWriter(buffer);
|
||||||
|
action(ref writer);
|
||||||
|
writer.Flush();
|
||||||
|
return buffer.WrittenSpan.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private delegate void WriterAction(ref MessagePackWriter writer);
|
||||||
|
|
||||||
|
// A store of two records of the given type, the marked one carrying the payload and the clean one an ordinary value.
|
||||||
|
private string Seed(CultDocumentRegistry registry, Type type, Func<object> clean, byte[] markedPayload, string name = "slots.cc")
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, name);
|
||||||
|
using (var cache = Open(path, registry))
|
||||||
|
{
|
||||||
|
cache.UpsertAsync(type, clean(), Marked).GetAwaiter().GetResult();
|
||||||
|
cache.UpsertAsync(type, clean(), Clean).GetAwaiter().GetResult();
|
||||||
|
cache.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||||
|
snapshot.Records.Single(record => record.Key == Marked.Value).Payload = markedPayload;
|
||||||
|
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
private string SeedSurface(string vector) =>
|
||||||
|
Seed(SurfaceRegistry, typeof(Surface), () => new Surface { Id = "x" }, VectorFiles.Committed("union-parity", vector));
|
||||||
|
|
||||||
|
private static CultStoredDocument Stored(CultCache cache, CultRecordKey key) => cache.GetStored(key)!;
|
||||||
|
|
||||||
|
// ---- a mark at every depth, with the type and both counts ----
|
||||||
|
|
||||||
|
[TestCase("extra-slot-record.msgpack", "UnionSurface", 8, 7, 1)]
|
||||||
|
[TestCase("extra-slot-nested-record.msgpack", nameof(UnionParityVectorTests.Composition), 3, 2, 1)]
|
||||||
|
[TestCase("extra-slot-arm.msgpack", nameof(UnionParityVectorTests.Box), 3, 2, 1)]
|
||||||
|
[TestCase("extra-slot-list-element.msgpack", nameof(UnionParityVectorTests.Text), 3, 2, 1)]
|
||||||
|
[TestCase("depth-skipped-at-limit.msgpack", "UnionSurface", 8, 7, 1)]
|
||||||
|
public void AMarkAtAnyDepthHoldsTheRecordReadOnly(string vector, string typeName, int slots, int declared, int occurrences)
|
||||||
|
{
|
||||||
|
var path = SeedSurface(vector);
|
||||||
|
using var cache = Open(path, SurfaceRegistry);
|
||||||
|
var unread = Stored(cache, Marked).UnreadSlots;
|
||||||
|
Assert.That(unread, Is.Not.Null, vector);
|
||||||
|
Assert.That(unread!.TypeName, Is.EqualTo(typeName), vector);
|
||||||
|
Assert.That(unread.SlotCount, Is.EqualTo(slots), vector);
|
||||||
|
Assert.That(unread.DeclaredCount, Is.EqualTo(declared), vector);
|
||||||
|
Assert.That(unread.Occurrences, Is.EqualTo(occurrences), vector);
|
||||||
|
Assert.That(unread.Explained, Is.False, "the reader's own id is not a compatible-id declaration: nothing explains the mark");
|
||||||
|
Assert.That(cache.IsWritable(Marked), Is.False, vector);
|
||||||
|
Assert.That(cache.Get<Surface>(Marked), Is.Not.Null, "a read-only record is still served");
|
||||||
|
Assert.That(Stored(cache, Clean).UnreadSlots, Is.Null);
|
||||||
|
Assert.That(cache.IsWritable(Clean), Is.True);
|
||||||
|
Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- every staged write and removal of a marked key is refused, nothing else is ----
|
||||||
|
|
||||||
|
private static void AssertSlotRefusal(TestDelegate act, string what, string type, int slots, int declared)
|
||||||
|
{
|
||||||
|
var refusal = Assert.Throws<CultSchemaConflictException>(act, what)!;
|
||||||
|
Assert.That(refusal.UnreadSlots, Is.Not.Null, what);
|
||||||
|
Assert.That(refusal.RecordKey, Is.EqualTo("marked"), what);
|
||||||
|
Assert.That(refusal.Message, Does.Contain("is read-only: " + type + " was read with " + slots + " slots where this build declares " + declared)
|
||||||
|
.And.Contain("record marked"), what);
|
||||||
|
Assert.That(refusal.Message, Does.Not.Contain(Extra), $"{what}: the refusal never echoes a slot value");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void EveryWriteAndRemovalOfAMarkedRecordIsRefusedAndAnotherKeyStillWrites()
|
||||||
|
{
|
||||||
|
var path = SeedSurface("extra-slot-nested-record.msgpack");
|
||||||
|
var before = File.ReadAllBytes(path);
|
||||||
|
using (var cache = Open(path, SurfaceRegistry))
|
||||||
|
{
|
||||||
|
var composition = nameof(UnionParityVectorTests.Composition);
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle<Surface>(Marked))), "upsert", composition, 3, 2);
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.UpsertVariant(Marked, Clean)), "variant", composition, 3, 2);
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "remove", composition, 3, 2);
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => { batch.Upsert(new Surface { Id = "other" }, new CultRecordHandle<Surface>(Clean)); batch.Remove(Marked); }), "mixed batch", composition, 3, 2);
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "refused writes leave the store as it was");
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "changed" }, new CultRecordHandle<Surface>(Clean))), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).Records.Count, Is.EqualTo(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- consent ----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ADeclaredIdDoesNotConsentToAnUnexplainedMark()
|
||||||
|
{
|
||||||
|
// The reader's own id is declared; the mark at depth 0 is in no compatible id's comparison: still read-only.
|
||||||
|
var path = SeedSurface("extra-slot-record.msgpack");
|
||||||
|
using var cache = Open(path, SurfaceRegistry);
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle<Surface>(Marked))), "depth 0 under the reader's own id", "UnionSurface", 8, 7);
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "removal", "UnionSurface", 8, 7);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every row of slot-outcomes.msgpack, from the committed files alone: the mark, Explained and IsWritable of each record of
|
||||||
|
// extra-slot-store.cc and extra-slot-consent.cc, and a removal of the key refused exactly where IsWritable is false.
|
||||||
|
[Test]
|
||||||
|
public void EverySlotOutcomeRowHoldsAndARemovalIsRefusedExactlyWhereTheRecordIsNotWritable()
|
||||||
|
{
|
||||||
|
var reader = new MessagePackReader(VectorFiles.Committed("union-parity", "slot-outcomes.msgpack"));
|
||||||
|
var rows = reader.ReadArrayHeader();
|
||||||
|
Assert.That(rows, Is.EqualTo(8));
|
||||||
|
for (var i = 0; i < rows; i++)
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(9));
|
||||||
|
var store = reader.ReadString()!;
|
||||||
|
var key = new CultRecordKey(reader.ReadString()!);
|
||||||
|
var writable = reader.ReadBoolean();
|
||||||
|
bool? explained = reader.TryReadNil() ? null : reader.ReadBoolean();
|
||||||
|
var type = reader.TryReadNil() ? null : reader.ReadString();
|
||||||
|
var counts = new int?[3];
|
||||||
|
for (var c = 0; c < 3; c++) counts[c] = reader.TryReadNil() ? null : reader.ReadInt32();
|
||||||
|
var (slots, declared, occurrences) = (counts[0], counts[1], counts[2]);
|
||||||
|
var flattenRefusedBy = reader.TryReadNil() ? null : reader.ReadString();
|
||||||
|
var row = $"{store} {key.Value}";
|
||||||
|
|
||||||
|
var path = Path.Combine(_directory, $"{i}-{store}");
|
||||||
|
File.WriteAllBytes(path, VectorFiles.Committed("union-parity", store));
|
||||||
|
var before = File.ReadAllBytes(path);
|
||||||
|
using var cache = Open(path, UnionParityVectorTests.SlotReader(store));
|
||||||
|
var unread = Stored(cache, key).UnreadSlots;
|
||||||
|
if (explained == null)
|
||||||
|
{
|
||||||
|
Assert.That(unread, Is.Null, row);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Assert.That(unread, Is.Not.Null, row);
|
||||||
|
Assert.That((unread!.Explained, unread.TypeName, unread.SlotCount, unread.DeclaredCount, unread.Occurrences),
|
||||||
|
Is.EqualTo((explained.Value, type, slots, declared, occurrences)), row);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(cache.IsWritable(key), Is.EqualTo(writable), row);
|
||||||
|
Assert.That(cache.IsWritable(Unheld), Is.True, "a key no store holds is writable");
|
||||||
|
if (flattenRefusedBy != null)
|
||||||
|
{
|
||||||
|
var refusal = Assert.Throws<CultSchemaConflictException>(() => cache.Commit(batch => batch.Flatten(key)), $"{row}: flatten")!;
|
||||||
|
Assert.That(refusal.RecordKey, Is.EqualTo(flattenRefusedBy), $"{row}: the refusal is the base's own");
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{row}: a refused flatten leaves the store as it was");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (writable)
|
||||||
|
{
|
||||||
|
Assert.That(cache.Commit(batch => batch.Remove(key)), Is.True, row);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var refusal = Assert.Throws<CultSchemaConflictException>(() => cache.Commit(batch => batch.Remove(key)), row)!;
|
||||||
|
Assert.That(refusal.UnreadSlots, Is.Not.Null, row);
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), $"{row}: a refused removal leaves the store as it was");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consent reaches only the slots the catalog comparison reports: the wide id's catalog reports slot 7, so a payload carrying
|
||||||
|
// slots 7 and 8 at depth 0 is not consented to, and a payload carrying slot 7 alone is.
|
||||||
|
[Test]
|
||||||
|
public void ConsentStopsAtTheSlotsTheCatalogReports()
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "extra-slot-consent.cc");
|
||||||
|
File.WriteAllBytes(path, VectorFiles.Committed("union-parity", "extra-slot-consent.cc"));
|
||||||
|
var before = File.ReadAllBytes(path);
|
||||||
|
using var cache = Open(path, UnionParityVectorTests.SlotReader("extra-slot-consent.cc"));
|
||||||
|
var over = new CultRecordHandle<UnionParityVectorTests.SurfaceDeclaring>(new CultRecordKey("over-consent"));
|
||||||
|
AssertWriteAgrees(cache, over.Key, () => cache.Commit(batch => batch.Upsert(new UnionParityVectorTests.SurfaceDeclaring { Id = "over" }, over)));
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "the refused write leaves the store as it was");
|
||||||
|
var consented = new CultRecordHandle<UnionParityVectorTests.SurfaceDeclaring>(new CultRecordKey("consented"));
|
||||||
|
AssertWriteAgrees(cache, consented.Key, () => cache.Commit(batch => batch.Upsert(new UnionParityVectorTests.SurfaceDeclaring { Id = "consented" }, consented)));
|
||||||
|
Assert.That(cache.IsWritable(consented.Key), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- carried verbatim ----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AMarkedRecordIsCarriedByteForByteWhenAnotherRecordIsWritten()
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "extra-slot-store.cc");
|
||||||
|
File.WriteAllBytes(path, VectorFiles.Committed("union-parity", "extra-slot-store.cc"));
|
||||||
|
var before = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||||
|
var original = before.Records.Single(record => record.Key == Marked.Value);
|
||||||
|
using (var cache = Open(path, SurfaceRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(cache.IsWritable(Marked), Is.False);
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "changed" }, new CultRecordHandle<Surface>(Clean))), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||||
|
var carried = after.Records.Single(record => record.Key == Marked.Value);
|
||||||
|
Assert.That(carried.Payload, Is.EqualTo(original.Payload));
|
||||||
|
Assert.That(carried.SchemaId, Is.EqualTo(original.SchemaId));
|
||||||
|
Assert.That(carried.StoredAt, Is.EqualTo(original.StoredAt));
|
||||||
|
Assert.That(after.SchemaCatalog.Select(CultDocumentMessagePackSerialization.SerializeCatalogEntry).Select(Convert.ToHexString),
|
||||||
|
Is.EqualTo(before.SchemaCatalog.Select(CultDocumentMessagePackSerialization.SerializeCatalogEntry).Select(Convert.ToHexString)));
|
||||||
|
Assert.That(after.Records.Single(record => record.Key == Clean.Value).Payload, Is.Not.EqualTo(before.Records.Single(record => record.Key == Clean.Value).Payload));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- slots short of the reader's, and gaps below the highest key, do not mark ----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void MissingTrailingSlotsDefaultAndAGapSlotDoesNotMark()
|
||||||
|
{
|
||||||
|
var short1 = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(1); w.Write("short"); });
|
||||||
|
var path = Seed(GapRegistry, typeof(GapDoc), () => new GapDoc { Name = "g" }, short1);
|
||||||
|
using (var cache = Open(path, GapRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(cache.Get<GapDoc>(Marked)!.Tail, Is.EqualTo(string.Empty), "a missing trailing slot keeps its default");
|
||||||
|
Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null);
|
||||||
|
Assert.That(cache.IsWritable(Marked), Is.True);
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(new GapDoc { Name = "short", Tail = "t" }, new CultRecordHandle<GapDoc>(Marked))), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
var gapped = Emit((ref MessagePackWriter w) => { w.WriteArrayHeader(4); w.Write("gap"); w.WriteNil(); w.Write("in a retired slot"); w.Write("tail"); });
|
||||||
|
var gapPath = Seed(GapRegistry, typeof(GapDoc), () => new GapDoc { Name = "g" }, gapped, "gap.cc");
|
||||||
|
using (var cache = Open(gapPath, GapRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null, "a non-nil value in a retired gap below the highest key does not mark");
|
||||||
|
Assert.That(cache.IsWritable(Marked), Is.True);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the catalog's consent still works, and is reported as explained ----
|
||||||
|
|
||||||
|
[CultDocument("tests.slot_wide", "tests.slot_wide.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class WideDoc
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Name = string.Empty;
|
||||||
|
[Key(1)] public string Colour = string.Empty;
|
||||||
|
[Key(2)] public string Extra = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
[CultDocument("tests.slot_wide", "tests.slot_wide.v2")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class NarrowDoc
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Name = string.Empty;
|
||||||
|
[Key(1)] public string Colour = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ADepthZeroSlotTheCatalogReportsIsExplainedAndTheUndeclaredRecordStaysUnwritable([Values] bool directory)
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, directory ? "store" : "store.cc");
|
||||||
|
CacheBackingStore Store() => directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path);
|
||||||
|
using (var writer = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(WideDoc) })))
|
||||||
|
{
|
||||||
|
writer.AddBackingStore(Store());
|
||||||
|
Assert.That(writer.Commit(batch => batch.Upsert(new WideDoc { Name = "w", Extra = "kept?" }, new CultRecordHandle<WideDoc>(Marked))), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
using var reader = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(NarrowDoc) }));
|
||||||
|
reader.AddBackingStore(Store());
|
||||||
|
var unread = Stored(reader, Marked).UnreadSlots;
|
||||||
|
Assert.That(unread, Is.Not.Null, "the store kind decodes through the marking path");
|
||||||
|
Assert.That(unread!.Explained, Is.True, "depth 0 and reported by the catalog comparison");
|
||||||
|
Assert.That(unread.TypeName, Is.EqualTo(nameof(NarrowDoc)));
|
||||||
|
Assert.That(unread.SlotCount, Is.EqualTo(3));
|
||||||
|
Assert.That(unread.DeclaredCount, Is.EqualTo(2));
|
||||||
|
Assert.That(reader.IsWritable(Marked), Is.False, "the wide id is undeclared: unowned, though the mark is explained");
|
||||||
|
Assert.Throws<CultSchemaConflictException>(() => reader.Commit(batch => batch.Remove(Marked)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- declared slots are the slots the type's own formatter writes ----
|
||||||
|
|
||||||
|
// MessagePack writes a non-public [Key] member of an AllowPrivate type: 2 slots, where a public-member count says 1.
|
||||||
|
[MessagePackObject(AllowPrivate = true)]
|
||||||
|
public sealed class PrivInner
|
||||||
|
{
|
||||||
|
[Key(0)] public string A = string.Empty;
|
||||||
|
[Key(1)] internal string B = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A [Key] getter that throws on an uninitialized instance; the parameterless constructor's instance is written instead.
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class Counted
|
||||||
|
{
|
||||||
|
private List<string> _items = new();
|
||||||
|
[Key(0)] public int Count { get => _items.Count; set => _items = Enumerable.Repeat(string.Empty, value).ToList(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same getter and no parameterless constructor: no instance can be written, so the guard leaves the type alone.
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class Uncounted
|
||||||
|
{
|
||||||
|
private readonly List<string> _items;
|
||||||
|
[SerializationConstructor] public Uncounted(int count) => _items = Enumerable.Repeat(string.Empty, count).ToList();
|
||||||
|
[Key(0)] public int Count => _items.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A type naming its own formatter owns its layout: one [Key] slot, but the formatter writes the item count and each item.
|
||||||
|
[MessagePackObject]
|
||||||
|
[MessagePackFormatter(typeof(BagFormatter))]
|
||||||
|
public sealed class Bag
|
||||||
|
{
|
||||||
|
[Key(0)] public List<string> Items = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class BagFormatter : IMessagePackFormatter<Bag?>
|
||||||
|
{
|
||||||
|
public void Serialize(ref MessagePackWriter writer, Bag? value, MessagePackSerializerOptions options)
|
||||||
|
{
|
||||||
|
if (value == null) { writer.WriteNil(); return; }
|
||||||
|
writer.WriteArrayHeader(1 + value.Items.Count);
|
||||||
|
writer.Write(value.Items.Count);
|
||||||
|
foreach (var item in value.Items) writer.Write(item);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Bag? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
|
||||||
|
{
|
||||||
|
if (reader.TryReadNil()) return null;
|
||||||
|
var count = reader.ReadArrayHeader();
|
||||||
|
reader.ReadInt32();
|
||||||
|
var bag = new Bag();
|
||||||
|
for (var i = 1; i < count; i++) bag.Items.Add(reader.ReadString()!);
|
||||||
|
return bag;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A [MessagePackObject] type an assembly resolver formats as [count, items...], with no [MessagePackFormatter] attribute: the
|
||||||
|
// resolver owns its layout.
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class ResolvedBag
|
||||||
|
{
|
||||||
|
[Key(0)] public List<string> Items = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class ResolvedBagResolver : IFormatterResolver
|
||||||
|
{
|
||||||
|
public static readonly IFormatterResolver Instance = new ResolvedBagResolver();
|
||||||
|
private static readonly IMessagePackFormatter<ResolvedBag?> Formatter = new ResolvedBagFormatter();
|
||||||
|
|
||||||
|
public IMessagePackFormatter<T>? GetFormatter<T>() => typeof(T) == typeof(ResolvedBag) ? (IMessagePackFormatter<T>)Formatter : null;
|
||||||
|
|
||||||
|
private sealed class ResolvedBagFormatter : IMessagePackFormatter<ResolvedBag?>
|
||||||
|
{
|
||||||
|
private static readonly BagFormatter Bag = new();
|
||||||
|
|
||||||
|
public void Serialize(ref MessagePackWriter writer, ResolvedBag? value, MessagePackSerializerOptions options) =>
|
||||||
|
Bag.Serialize(ref writer, value == null ? null : new Bag { Items = value.Items }, options);
|
||||||
|
|
||||||
|
public ResolvedBag? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) =>
|
||||||
|
Bag.Deserialize(ref reader, options) is { } bag ? new ResolvedBag { Items = bag.Items } : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A type an assembly resolver claims by handing out MessagePack's own [Key]-layout formatter: still measured.
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class Relayed
|
||||||
|
{
|
||||||
|
[Key(0)] public string A = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class RelayResolver : IFormatterResolver
|
||||||
|
{
|
||||||
|
public static readonly IFormatterResolver Instance = new RelayResolver();
|
||||||
|
|
||||||
|
public IMessagePackFormatter<T>? GetFormatter<T>() =>
|
||||||
|
typeof(T) == typeof(Relayed) ? global::MessagePack.Resolvers.StandardResolver.Instance.GetFormatter<T>() : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A [MessagePackObject] collection: MessagePack writes it as its items, not by [Key].
|
||||||
|
[MessagePackObject]
|
||||||
|
public class TagList : List<string> { }
|
||||||
|
|
||||||
|
[CultDocument("tests.slot_measured", "tests.slot_measured.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class MeasuredDoc
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Name = string.Empty;
|
||||||
|
[Key(1)] public PrivInner? Private;
|
||||||
|
[Key(2)] public Counted? Counted;
|
||||||
|
[Key(3)] public Uncounted? Uncounted;
|
||||||
|
[Key(4)] public Bag? Bag;
|
||||||
|
[Key(5)] public ResolvedBag? Resolved;
|
||||||
|
[Key(6)] public TagList? Tags;
|
||||||
|
[Key(7)] public Relayed? Relayed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly CultDocumentRegistry MeasuredRegistry = CultDocumentRegistry.ForTypes(new[] { typeof(MeasuredDoc) });
|
||||||
|
|
||||||
|
private static MeasuredDoc Measured(string member) => member switch
|
||||||
|
{
|
||||||
|
"private" => new MeasuredDoc { Name = member, Private = new PrivInner { A = "a", B = "bee" } },
|
||||||
|
"constructed" => new MeasuredDoc { Name = member, Counted = new Counted { Count = 2 } },
|
||||||
|
"unmeasurable" => new MeasuredDoc { Name = member, Uncounted = new Uncounted(2) },
|
||||||
|
"own-formatter" => new MeasuredDoc { Name = member, Bag = new Bag { Items = { "x", "y", "z" } } },
|
||||||
|
"resolver-formatted" => new MeasuredDoc { Name = member, Resolved = new ResolvedBag { Items = { "x", "y", "z" } } },
|
||||||
|
"collection" => new MeasuredDoc { Name = member, Tags = new TagList { "a", "b", "c" } },
|
||||||
|
_ => throw new ArgumentOutOfRangeException(nameof(member), member, null)
|
||||||
|
};
|
||||||
|
|
||||||
|
[TestCase("private", TestName = "APrivateKeyedMemberDoesNotMark")]
|
||||||
|
[TestCase("constructed", TestName = "ATypeMeasuredThroughItsConstructorDoesNotMark")]
|
||||||
|
[TestCase("unmeasurable", TestName = "AnUnmeasurableTypeIsNotGuarded")]
|
||||||
|
[TestCase("own-formatter", TestName = "ATypeWithItsOwnFormatterIsNotMeasuredByKeys")]
|
||||||
|
[TestCase("resolver-formatted", TestName = "ATypeAResolverFormatsIsNotMeasuredByKeys")]
|
||||||
|
[TestCase("collection", TestName = "AMessagePackObjectCollectionIsNotMeasuredByKeys")]
|
||||||
|
public void ARecordWrittenByItsOwnTypeDoesNotMark(string member)
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "measured.cc");
|
||||||
|
using (var cache = Open(path, MeasuredRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(Measured(member), new CultRecordHandle<MeasuredDoc>(Marked))), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
using (var cache = Open(path, MeasuredRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(Stored(cache, Marked).UnreadSlots, Is.Null, member);
|
||||||
|
Assert.That(cache.IsWritable(Marked), Is.True, member);
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(Measured(member), new CultRecordHandle<MeasuredDoc>(Marked))), Is.True, member);
|
||||||
|
Assert.That(cache.Commit(batch => batch.Remove(Marked)), Is.True, member);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One slot past what the formatter writes still marks, with the written count as declared.
|
||||||
|
[Test]
|
||||||
|
public void AnExtraSlotPastTheWrittenCountStillMarks()
|
||||||
|
{
|
||||||
|
var privatePayload = Emit((ref MessagePackWriter w) =>
|
||||||
|
{
|
||||||
|
w.WriteArrayHeader(5); w.Write("p");
|
||||||
|
w.WriteArrayHeader(3); w.Write("a"); w.Write("bee"); w.Write(Extra);
|
||||||
|
w.WriteNil(); w.WriteNil(); w.WriteNil();
|
||||||
|
});
|
||||||
|
var path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, privatePayload, "private.cc");
|
||||||
|
using (var cache = Open(path, MeasuredRegistry))
|
||||||
|
{
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "AllowPrivate type", nameof(PrivInner), 3, 2);
|
||||||
|
Assert.That(cache.IsWritable(Clean), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
var countedPayload = Emit((ref MessagePackWriter w) =>
|
||||||
|
{
|
||||||
|
w.WriteArrayHeader(5); w.Write("c"); w.WriteNil();
|
||||||
|
w.WriteArrayHeader(2); w.Write(0); w.Write(Extra);
|
||||||
|
w.WriteNil(); w.WriteNil();
|
||||||
|
});
|
||||||
|
path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, countedPayload, "counted.cc");
|
||||||
|
using (var cache = Open(path, MeasuredRegistry))
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "measured through its constructor", nameof(Counted), 2, 1);
|
||||||
|
|
||||||
|
// A resolver that hands out MessagePack's own [Key] formatter exempts nothing.
|
||||||
|
var relayedPayload = Emit((ref MessagePackWriter w) =>
|
||||||
|
{
|
||||||
|
w.WriteArrayHeader(8); w.Write("r");
|
||||||
|
for (var i = 0; i < 6; i++) w.WriteNil();
|
||||||
|
w.WriteArrayHeader(2); w.Write("a"); w.Write(Extra);
|
||||||
|
});
|
||||||
|
path = Seed(MeasuredRegistry, typeof(MeasuredDoc), () => new MeasuredDoc { Name = "c" }, relayedPayload, "relayed.cc");
|
||||||
|
using (var cache = Open(path, MeasuredRegistry))
|
||||||
|
AssertSlotRefusal(() => cache.Commit(batch => batch.Remove(Marked)), "relayed by a consumer resolver", nameof(Relayed), 2, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- a flatten writes its base chain's decoded view: refused when a record on the chain is not owned ----
|
||||||
|
|
||||||
|
private static readonly CultRecordKey Base = new("base");
|
||||||
|
private static readonly CultRecordKey Var = new("var");
|
||||||
|
private static readonly CultRecordKey Fresh = new("fresh");
|
||||||
|
|
||||||
|
// A store holding plain record base and its variant var, base then given the payload.
|
||||||
|
private string SeedVariant(byte[]? basePayload, string name)
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, name);
|
||||||
|
using (var cache = Open(path, SurfaceRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "base" }, new CultRecordHandle<Surface>(Base))), Is.True);
|
||||||
|
Assert.That(cache.Commit(batch => batch.UpsertVariant(Var, Base, Named(cache, Var))), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (basePayload == null) return path;
|
||||||
|
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||||
|
snapshot.Records.Single(record => record.Key == Base.Value).Payload = basePayload;
|
||||||
|
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A variant overrides its name.
|
||||||
|
private static CultVariantOverride[] Named(CultCache cache, CultRecordKey key) => new[] { cache.Override<Surface>(nameof(Surface.Id), key.Value) };
|
||||||
|
|
||||||
|
private static void AssertBaseRefusal(TestDelegate act, string what, bool unreadSlots)
|
||||||
|
{
|
||||||
|
var refusal = Assert.Throws<CultSchemaConflictException>(act, what)!;
|
||||||
|
Assert.That(refusal.RecordKey, Is.EqualTo(Base.Value), $"{what}: the refusal is the base's own");
|
||||||
|
Assert.That(refusal.UnreadSlots, unreadSlots ? Is.Not.Null : Is.Null, what);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AFlattenThroughABaseHeldForUnreadSlotsIsRefusedAndTheVariantStillWritesItsDelta()
|
||||||
|
{
|
||||||
|
var nested = VectorFiles.Committed("union-parity", "extra-slot-nested-record.msgpack");
|
||||||
|
var path = SeedVariant(nested, "flatten.cc");
|
||||||
|
var before = File.ReadAllBytes(path);
|
||||||
|
using (var cache = Open(path, SurfaceRegistry))
|
||||||
|
{
|
||||||
|
Assert.That(cache.IsWritable(Base), Is.False);
|
||||||
|
Assert.That(cache.IsWritable(Var), Is.True, "the variant persists only its delta");
|
||||||
|
AssertBaseRefusal(() => cache.Commit(batch => batch.Flatten(Var)), "Commit(Flatten)", unreadSlots: true);
|
||||||
|
AssertBaseRefusal(() => cache.FlattenAsync(Var).GetAwaiter().GetResult(), "FlattenAsync", unreadSlots: true);
|
||||||
|
AssertBaseRefusal(() => cache.Commit(batch => { batch.UpsertVariant(Fresh, Base, Named(cache, Fresh)); batch.Flatten(Fresh); }), "a new variant flattened as it lands", unreadSlots: true);
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before), "refused flattens leave the store as it was");
|
||||||
|
Assert.That(cache.Commit(batch => batch.UpsertVariant(Var, Base, Named(cache, Var))), Is.True, "a variant write persists only its delta");
|
||||||
|
Assert.That(cache.Commit(batch => batch.Remove(Var)), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
var after = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
|
||||||
|
Assert.That(after.Records.Single(record => record.Key == Base.Value).Payload, Is.EqualTo(nested), "the base's bytes stay");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void AFlattenThroughABaseUnderAnUndeclaredDroppingIdIsRefused()
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "drift.cc");
|
||||||
|
using (var writer = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(WideDoc) })))
|
||||||
|
Assert.That(writer.Commit(batch => batch.Upsert(new WideDoc { Name = "base", Extra = "kept?" }, new CultRecordHandle<WideDoc>(Base))), Is.True);
|
||||||
|
|
||||||
|
using var reader = Open(path, CultDocumentRegistry.ForTypes(new[] { typeof(NarrowDoc) }));
|
||||||
|
Assert.That(reader.IsWritable(Base), Is.False);
|
||||||
|
Assert.That(reader.Commit(batch => batch.UpsertVariant(Var, Base, new[] { reader.Override<NarrowDoc>(nameof(NarrowDoc.Name), Var.Value) })), Is.True, "a variant over the unowned base persists only its delta");
|
||||||
|
var before = File.ReadAllBytes(path);
|
||||||
|
AssertBaseRefusal(() => reader.Commit(batch => batch.Flatten(Var)), "flatten over undeclared drift", unreadSlots: false);
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(before));
|
||||||
|
Assert.That(reader.Commit(batch => batch.Remove(Var)), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- IsWritable is the write's own answer ----
|
||||||
|
|
||||||
|
// False exactly where the write throws.
|
||||||
|
private static void AssertWriteAgrees(CultCache cache, CultRecordKey key, Action write)
|
||||||
|
{
|
||||||
|
var writable = cache.IsWritable(key);
|
||||||
|
Exception? thrown = null;
|
||||||
|
try { write(); }
|
||||||
|
catch (InvalidOperationException exception) { thrown = exception; }
|
||||||
|
Assert.That(thrown == null, Is.EqualTo(writable), $"IsWritable({key.Value}) is {writable}; the write {(thrown == null ? "landed" : "threw: " + thrown.Message)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void IsWritableIsFalseInAReadOnlyStoreAndAFlattenThereIsNotRefusedForItsCleanBase()
|
||||||
|
{
|
||||||
|
var path = SeedVariant(null, "readonly.cc");
|
||||||
|
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = SurfaceRegistry, ReadOnly = true });
|
||||||
|
Assert.That(cache.IsWritable(Base), Is.False);
|
||||||
|
AssertWriteAgrees(cache, Base, () => cache.Commit(batch => batch.Upsert(new Surface { Id = "new" }, new CultRecordHandle<Surface>(Base))));
|
||||||
|
AssertWriteAgrees(cache, Var, () => cache.Commit(batch => batch.Remove(Var)));
|
||||||
|
var refusal = Assert.Throws<InvalidOperationException>(() => cache.Commit(batch => batch.Flatten(Var)))!;
|
||||||
|
Assert.That(refusal, Is.Not.InstanceOf<CultSchemaConflictException>(), "nothing on the chain is lost: the refusal is the store's read-only one");
|
||||||
|
Assert.That(refusal.Message, Does.Contain("read-only backing store"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// A key the cache does not hold, but a store carries as a foreign record: the store's removal check refuses it.
|
||||||
|
[Test]
|
||||||
|
public void IsWritableIsFalseForAnUnheldKeyAStoreListsForeign()
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "foreign.cc");
|
||||||
|
using (var writer = Open(path, GapRegistry))
|
||||||
|
Assert.That(writer.Commit(batch => batch.Upsert(new GapDoc { Name = "g" }, new CultRecordHandle<GapDoc>(Marked))), Is.True);
|
||||||
|
|
||||||
|
using var cache = Open(path, SurfaceRegistry);
|
||||||
|
Assert.That(cache.GetStored(Marked), Is.Null, "the cache does not hold a record no registered type reads");
|
||||||
|
Assert.That(cache.IsWritable(Marked), Is.False);
|
||||||
|
AssertWriteAgrees(cache, Marked, () => cache.Commit(batch => batch.Remove(Marked)));
|
||||||
|
Assert.That(cache.IsWritable(Unheld), Is.True);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void IsWritableIsFalseForARecordNoAttachedStoreIsHomeTo()
|
||||||
|
{
|
||||||
|
using var cache = new CultCache(CultDocumentRegistry.ForTypes(new[] { typeof(Surface), typeof(GapDoc) }));
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(new Surface { Id = "memory" }, new CultRecordHandle<Surface>(Clean))), Is.True);
|
||||||
|
Assert.That(cache.IsWritable(Clean), Is.True, "a store-less cache writes its own records");
|
||||||
|
cache.AddBackingStore(new SingleFileMessagePackBackingStore(Path.Combine(_directory, "gaps.cc")), typeof(GapDoc));
|
||||||
|
Assert.That(cache.IsWritable(Clean), Is.False);
|
||||||
|
AssertWriteAgrees(cache, Clean, () => cache.Commit(batch => batch.Upsert(new Surface { Id = "x" }, new CultRecordHandle<Surface>(Clean))));
|
||||||
|
AssertWriteAgrees(cache, Clean, () => cache.Commit(batch => batch.Remove(Clean)));
|
||||||
|
Assert.That(cache.IsWritable(Unheld), Is.True, "an unheld key no store lists foreign is writable");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -7,6 +7,7 @@ using System.Linq;
|
||||||
using GameCult.Caching.MessagePack;
|
using GameCult.Caching.MessagePack;
|
||||||
using MessagePack;
|
using MessagePack;
|
||||||
using NUnit.Framework;
|
using NUnit.Framework;
|
||||||
|
using NUnit.Framework.Constraints;
|
||||||
using static GameCult.Caching.Tests.ElementIdTests;
|
using static GameCult.Caching.Tests.ElementIdTests;
|
||||||
|
|
||||||
namespace GameCult.Caching.Tests
|
namespace GameCult.Caching.Tests
|
||||||
|
|
@ -61,6 +62,39 @@ namespace GameCult.Caching.Tests
|
||||||
.Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries))
|
.Select(line => line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries))
|
||||||
.Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}"));
|
.Select(cells => new TestCaseData(cells[0], cells[CSharp] == "reads").SetName($"{cells[0]} {cells[CSharp]}"));
|
||||||
|
|
||||||
|
// The row `@directory` has no bytes: the walker makes an empty directory at the store path, and a runtime must refuse it untouched.
|
||||||
|
private const string DirectoryRow = "@directory";
|
||||||
|
|
||||||
|
private static byte[] Place(string path, string vector)
|
||||||
|
{
|
||||||
|
if (vector != DirectoryRow)
|
||||||
|
{
|
||||||
|
var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
|
||||||
|
File.WriteAllBytes(path, bytes);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (File.Exists(path))
|
||||||
|
File.Delete(path);
|
||||||
|
Directory.CreateDirectory(path);
|
||||||
|
return Array.Empty<byte>();
|
||||||
|
}
|
||||||
|
|
||||||
|
// A directory at the store path is refused as IOException ("not a store file", CultCache.cs): not a store that is unreadable.
|
||||||
|
private static IResolveConstraint Refusal(string vector) =>
|
||||||
|
vector == DirectoryRow ? Throws.TypeOf<IOException>() : Throws.TypeOf<CultStoreUnreadableException>();
|
||||||
|
|
||||||
|
private static void AssertUntouched(string path, string vector, byte[] bytes, string message)
|
||||||
|
{
|
||||||
|
if (vector == DirectoryRow)
|
||||||
|
{
|
||||||
|
Assert.That(Directory.Exists(path), message);
|
||||||
|
Assert.That(Directory.EnumerateFileSystemEntries(path), Is.Empty, message);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), message);
|
||||||
|
}
|
||||||
|
|
||||||
private static bool HoldsAVariant(byte[] bytes) =>
|
private static bool HoldsAVariant(byte[] bytes) =>
|
||||||
CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes).Records.Any(record => record.Variant != null);
|
CultDocumentMessagePackSerialization.DeserializeSnapshot(bytes).Records.Any(record => record.Variant != null);
|
||||||
|
|
||||||
|
|
@ -74,18 +108,33 @@ namespace GameCult.Caching.Tests
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ADuplicateKeyIsRefusedNamingTheKeyAndTheFile()
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "dup.cc");
|
||||||
|
File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), "dup-key-same-schema-v3.bin")));
|
||||||
|
|
||||||
|
Assert.That(() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }),
|
||||||
|
Throws.TypeOf<CultStoreUnreadableException>()
|
||||||
|
.With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path)
|
||||||
|
.And.Message.Contains("more than once"));
|
||||||
|
}
|
||||||
|
|
||||||
[TestCaseSource(nameof(Vectors))]
|
[TestCaseSource(nameof(Vectors))]
|
||||||
public void OpenReadsExactlyTheFilesTheVectorsSayItReads(string vector, bool reads)
|
public void OpenReadsExactlyTheFilesTheVectorsSayItReads(string vector, bool reads)
|
||||||
{
|
{
|
||||||
var path = Path.Combine(_directory, "open.cc");
|
var path = Path.Combine(_directory, "open.cc");
|
||||||
File.WriteAllBytes(path, File.ReadAllBytes(Path.Combine(VectorRoot(), vector)));
|
var bytes = Place(path, vector);
|
||||||
|
|
||||||
void Open() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }).Dispose();
|
void Open() => CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry }).Dispose();
|
||||||
|
|
||||||
if (reads)
|
if (reads)
|
||||||
Assert.DoesNotThrow(Open);
|
Assert.DoesNotThrow(Open);
|
||||||
else
|
else
|
||||||
Assert.That(Open, Throws.TypeOf<CultStoreUnreadableException>().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path));
|
{
|
||||||
|
Assert.That(Open, vector == DirectoryRow ? Refusal(vector) : Throws.TypeOf<CultStoreUnreadableException>().With.Property(nameof(CultStoreUnreadableException.Path)).EqualTo(path));
|
||||||
|
AssertUntouched(path, vector, bytes, "a store this runtime cannot open was changed by the attempt");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestCaseSource(nameof(Vectors))]
|
[TestCaseSource(nameof(Vectors))]
|
||||||
|
|
@ -93,8 +142,7 @@ namespace GameCult.Caching.Tests
|
||||||
{
|
{
|
||||||
var path = Seed("flush.cc");
|
var path = Seed("flush.cc");
|
||||||
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
|
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
|
||||||
var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
|
var bytes = Place(path, vector);
|
||||||
File.WriteAllBytes(path, bytes);
|
|
||||||
var store = cache.BackingStores[0];
|
var store = cache.BackingStores[0];
|
||||||
|
|
||||||
if (reads && HoldsAVariant(bytes))
|
if (reads && HoldsAVariant(bytes))
|
||||||
|
|
@ -114,8 +162,8 @@ namespace GameCult.Caching.Tests
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.That(() => store.PushAll(), Throws.TypeOf<CultStoreUnreadableException>());
|
Assert.That(() => store.PushAll(), Refusal(vector));
|
||||||
Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten");
|
AssertUntouched(path, vector, bytes, "a file this runtime cannot read was rewritten");
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestCaseSource(nameof(Vectors))]
|
[TestCaseSource(nameof(Vectors))]
|
||||||
|
|
@ -123,8 +171,7 @@ namespace GameCult.Caching.Tests
|
||||||
{
|
{
|
||||||
var path = Seed("commit.cc");
|
var path = Seed("commit.cc");
|
||||||
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
|
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
|
||||||
var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
|
var bytes = Place(path, vector);
|
||||||
File.WriteAllBytes(path, bytes);
|
|
||||||
|
|
||||||
if (reads && HoldsAVariant(bytes))
|
if (reads && HoldsAVariant(bytes))
|
||||||
{
|
{
|
||||||
|
|
@ -141,8 +188,8 @@ namespace GameCult.Caching.Tests
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Throws.TypeOf<CultStoreUnreadableException>());
|
Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))), Refusal(vector));
|
||||||
Assert.That(File.ReadAllBytes(path), Is.EqualTo(bytes), "a file this runtime cannot read was rewritten");
|
AssertUntouched(path, vector, bytes, "a file this runtime cannot read was rewritten");
|
||||||
}
|
}
|
||||||
|
|
||||||
private string DirectoryStore(string name)
|
private string DirectoryStore(string name)
|
||||||
|
|
@ -267,6 +314,9 @@ namespace GameCult.Caching.Tests
|
||||||
Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))),
|
Assert.That(() => cache.Commit(batch => batch.Upsert(typeof(IdDeck), new IdDeck { Name = "e" }, new CultRecordKey("e"))),
|
||||||
Throws.TypeOf<IOException>().With.Message.Contains("directory"));
|
Throws.TypeOf<IOException>().With.Message.Contains("directory"));
|
||||||
|
|
||||||
|
Assert.That(() => cache.Commit(batch => batch.Remove(new CultRecordKey("e"))),
|
||||||
|
Throws.TypeOf<IOException>().With.Message.Contains("directory"));
|
||||||
|
|
||||||
Assert.That(Directory.GetFileSystemEntries(path), Is.Empty);
|
Assert.That(Directory.GetFileSystemEntries(path), Is.Empty);
|
||||||
Assert.That(Directory.GetFileSystemEntries(later), Is.Empty);
|
Assert.That(Directory.GetFileSystemEntries(later), Is.Empty);
|
||||||
}
|
}
|
||||||
|
|
@ -667,7 +717,7 @@ namespace GameCult.Caching.Tests
|
||||||
[Test]
|
[Test]
|
||||||
public void EveryVectorInTheFolderHasAManifestRow()
|
public void EveryVectorInTheFolderHasAManifestRow()
|
||||||
{
|
{
|
||||||
var rows = Vectors().Select(row => (string)row.Arguments[0]).Where(name => !name.StartsWith("..", StringComparison.Ordinal)).OrderBy(name => name, StringComparer.Ordinal);
|
var rows = Vectors().Select(row => (string)row.Arguments[0]).Where(name => !name.StartsWith("..", StringComparison.Ordinal) && name != DirectoryRow).OrderBy(name => name, StringComparer.Ordinal);
|
||||||
var files = Directory.GetFiles(VectorRoot(), "*.bin").Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal);
|
var files = Directory.GetFiles(VectorRoot(), "*.bin").Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal);
|
||||||
Assert.That(rows, Is.EqualTo(files));
|
Assert.That(rows, Is.EqualTo(files));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -252,7 +252,7 @@ namespace GameCult.Caching.Tests
|
||||||
private static Emit ImageArm(string id, string uri, string alt) => ArmOf(5, Str(id), Str(uri), Nil, Str(alt));
|
private static Emit ImageArm(string id, string uri, string alt) => ArmOf(5, Str(id), Str(uri), Nil, Str(alt));
|
||||||
|
|
||||||
// The constructed value's payload, with a part of the structure replaceable to build a refusal or an extra-slot case.
|
// The constructed value's payload, with a part of the structure replaceable to build a refusal or an extra-slot case.
|
||||||
private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, Emit? trailing = null)
|
private static byte[] Surface(Emit? root = null, Emit? outcomes = null, Emit? crafted = null, Extra? extra = null, params Emit[] trailing)
|
||||||
{
|
{
|
||||||
extra ??= new Extra();
|
extra ??= new Extra();
|
||||||
root ??= BoxArm("0000000000c0", Slots(
|
root ??= BoxArm("0000000000c0", Slots(
|
||||||
|
|
@ -264,7 +264,7 @@ namespace GameCult.Caching.Tests
|
||||||
outcomes ??= Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3")));
|
outcomes ??= Slots(ArmOf(0, Str("wait"), Str("0000000000a1")), ArmOf(1, Num(7), Str("0000000000a2")), ArmOf(2, Str("0000000000a3")));
|
||||||
crafted ??= ArmOf(1, Str("gear"), Num(4));
|
crafted ??= ArmOf(1, Str("gear"), Num(4));
|
||||||
var slots = new[] { Str("surface"), Num(3), root, outcomes, ArmOf(1, Str("hammer"), Num(12)), crafted, Nil };
|
var slots = new[] { Str("surface"), Num(3), root, outcomes, ArmOf(1, Str("hammer"), Num(12)), crafted, Nil };
|
||||||
return Raw(Slots(trailing != null ? slots.Append(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots));
|
return Raw(Slots(trailing.Length > 0 ? slots.Concat(trailing).ToArray() : extra.Record ? slots.Append(ExtraSlot).ToArray() : slots));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static byte[] SerializeValue(UnionSurface value) => CultDocumentMessagePackSerialization.SerializeUntyped(value, typeof(UnionSurface), Registry);
|
private static byte[] SerializeValue(UnionSurface value) => CultDocumentMessagePackSerialization.SerializeUntyped(value, typeof(UnionSurface), Registry);
|
||||||
|
|
@ -338,6 +338,73 @@ namespace GameCult.Caching.Tests
|
||||||
[MessagePackObject] public sealed class Floor2 : UnitOutcome { }
|
[MessagePackObject] public sealed class Floor2 : UnitOutcome { }
|
||||||
[MessagePackObject] public sealed class Ceiling2 : UnitOutcome { }
|
[MessagePackObject] public sealed class Ceiling2 : UnitOutcome { }
|
||||||
|
|
||||||
|
// The document a unit arm is placed in for a store: [Name, Outcome].
|
||||||
|
[CultDocument("tests.slot_unit", "tests.slot_unit.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class UnitDoc
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Name = string.Empty;
|
||||||
|
[Key(1)] public UnitOutcome? Outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the slot stores: extra-slot-store.cc, extra-slot-consent.cc and their outcomes as data ----
|
||||||
|
|
||||||
|
// The id extra-slot-consent.cc's records are stored under: UnionSurface's catalog entry with one more member, slot 7.
|
||||||
|
internal const string WideId = "tests.union_surface.wide";
|
||||||
|
|
||||||
|
// UnionSurface's members under a new version, listing the wide id compatible: the consent vector's reader.
|
||||||
|
[CultDocument("tests.union_surface", "tests.union_surface.v2", CompatibleSchemaIds = new[] { WideId })]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class SurfaceDeclaring
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Id = string.Empty;
|
||||||
|
[Key(1)] public int Version;
|
||||||
|
[Key(2)] public Node? Root;
|
||||||
|
[Key(3)] public List<Outcome> Outcomes = new();
|
||||||
|
[Key(4)] public Item? Part;
|
||||||
|
[Key(5)] public CraftedItem? Crafted;
|
||||||
|
[Key(6)] public Node? Missing;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The registry each slot store is read with.
|
||||||
|
internal static CultDocumentRegistry SlotReader(string store) => store switch
|
||||||
|
{
|
||||||
|
"extra-slot-store.cc" => CultDocumentRegistry.ForTypes(new[] { typeof(UnionSurface), typeof(UnitDoc) }),
|
||||||
|
"extra-slot-consent.cc" => CultDocumentRegistry.ForTypes(new[] { typeof(SurfaceDeclaring) }),
|
||||||
|
_ => throw new ArgumentOutOfRangeException(nameof(store), store, null)
|
||||||
|
};
|
||||||
|
|
||||||
|
// One row of slot-outcomes.msgpack: what the store's reader holds at the key. The mark fields are null for an unmarked record;
|
||||||
|
// FlattenRefusedBy is the key whose refusal refuses a Flatten of the record, null when the record is no variant or flattens.
|
||||||
|
private sealed record SlotOutcome(string Store, string Key, bool Writable, bool? Explained, string? Type, int? Slots, int? Declared, int? Occurrences, string? FlattenRefusedBy = null);
|
||||||
|
|
||||||
|
private static readonly SlotOutcome[] SlotOutcomes =
|
||||||
|
{
|
||||||
|
new("extra-slot-store.cc", "marked", false, false, nameof(Composition), 3, 2, 1),
|
||||||
|
new("extra-slot-store.cc", "clean", true, null, null, null, null, null),
|
||||||
|
new("extra-slot-store.cc", "unit-arm", false, false, nameof(Denied2), 1, 0, 1),
|
||||||
|
new("extra-slot-store.cc", "marked-variant", true, null, null, null, null, null, "marked"),
|
||||||
|
new("extra-slot-consent.cc", "consented", true, true, nameof(SurfaceDeclaring), 8, 7, 1),
|
||||||
|
new("extra-slot-consent.cc", "over-consent", false, false, nameof(SurfaceDeclaring), 9, 7, 1),
|
||||||
|
new("extra-slot-consent.cc", "nested", false, false, nameof(SurfaceDeclaring), 8, 7, 2),
|
||||||
|
new("extra-slot-consent.cc", "clean", true, null, null, null, null, null),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static byte[] SlotOutcomesFile() => Raw((ref MessagePackWriter w) =>
|
||||||
|
{
|
||||||
|
w.WriteArrayHeader(SlotOutcomes.Length);
|
||||||
|
foreach (var row in SlotOutcomes)
|
||||||
|
{
|
||||||
|
w.WriteArrayHeader(9);
|
||||||
|
w.Write(row.Store); w.Write(row.Key); w.Write(row.Writable);
|
||||||
|
if (row.Explained is { } explained) w.Write(explained); else w.WriteNil();
|
||||||
|
if (row.Type is { } type) w.Write(type); else w.WriteNil();
|
||||||
|
foreach (var count in new[] { row.Slots, row.Declared, row.Occurrences })
|
||||||
|
if (count is { } value) w.Write(value); else w.WriteNil();
|
||||||
|
if (row.FlattenRefusedBy is { } refusedBy) w.Write(refusedBy); else w.WriteNil();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
private static MessagePackSerializerOptions UnitOptions => CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly);
|
private static MessagePackSerializerOptions UnitOptions => CultDocumentMessagePackSerialization.OptionsFor(typeof(UnionSurface).Assembly);
|
||||||
|
|
||||||
// The reference writes a member-less arm as [key, []]; the other runtimes read these files and write the same bytes.
|
// The reference writes a member-less arm as [key, []]; the other runtimes read these files and write the same bytes.
|
||||||
|
|
@ -413,6 +480,79 @@ namespace GameCult.Caching.Tests
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A single-file store of the records, each written as its type's value with the store's clock pinned as union-surface.cc's, then
|
||||||
|
// its payload replaced by bytes no current writer produces; each variant is written over its base (overriding the name to its
|
||||||
|
// key) before the payloads are replaced; edit changes the snapshot further before it is serialized.
|
||||||
|
private static byte[] SlotStore(CultDocumentRegistry registry, (string Key, object Value, byte[] Payload)[] records, Action<CultPersistedStoreSnapshot>? edit = null,
|
||||||
|
params (string Key, string Base)[] variants)
|
||||||
|
{
|
||||||
|
var work = Path.Combine(Path.GetTempPath(), $"cultlib-unionparity-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(work);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var scratch = Path.Combine(work, "slots.cc");
|
||||||
|
using (var cache = CultCacheMessagePack.Create(scratch, new CultCacheOpenOptions { Registry = registry, StoreFlushOnDispose = true }))
|
||||||
|
{
|
||||||
|
cache.Clock = () => StoredAt;
|
||||||
|
foreach (var (key, value, _) in records)
|
||||||
|
cache.UpsertAsync(value.GetType(), value, new CultRecordKey(key)).GetAwaiter().GetResult();
|
||||||
|
foreach (var (key, baseKey) in variants)
|
||||||
|
Assert.That(cache.Commit(batch => batch.UpsertVariant(new CultRecordKey(key), new CultRecordKey(baseKey),
|
||||||
|
new[] { cache.Override(typeof(UnionSurface), nameof(UnionSurface.Id), key) })), Is.True);
|
||||||
|
cache.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(scratch));
|
||||||
|
foreach (var (key, _, payload) in records)
|
||||||
|
snapshot.Records.Single(record => record.Key == key).Payload = payload;
|
||||||
|
edit?.Invoke(snapshot);
|
||||||
|
return CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(work, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The store the slot-readonly tests open and the other runtimes read: "marked" holds the extra-slot-nested-record payload,
|
||||||
|
// "clean" the ordinary value, "unit-arm" a tests.slot_unit document ["unit-arm", [2, ["extra-slot"]]], and "marked-variant" a
|
||||||
|
// variant of "marked" overriding only its Id: writable, but a flatten of it is refused for its base.
|
||||||
|
[Test]
|
||||||
|
public void ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord()
|
||||||
|
{
|
||||||
|
Check("extra-slot-store.cc", SlotStore(SlotReader("extra-slot-store.cc"), new (string, object, byte[])[]
|
||||||
|
{
|
||||||
|
("marked", Constructed(), Committed("extra-slot-nested-record.msgpack")),
|
||||||
|
("clean", Constructed(), Surface()),
|
||||||
|
("unit-arm", new UnitDoc { Name = "unit-arm", Outcome = new Denied2() }, Raw(Slots(Str("unit-arm"), ArmOf(2, ExtraSlot)))),
|
||||||
|
}, null, ("marked-variant", "marked")));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consent store: every record under the wide id, whose catalog entry is UnionSurface's 7 members plus slot 7. Its reader
|
||||||
|
// lists the wide id compatible, so the comparison reports slot 7 ignored; only a depth-0 mark within that slot is consented to.
|
||||||
|
[Test]
|
||||||
|
public void ExtraSlotConsentVectorHoldsTheConsentCases()
|
||||||
|
{
|
||||||
|
Check("extra-slot-consent.cc", SlotStore(Registry, new (string, object, byte[])[]
|
||||||
|
{
|
||||||
|
("consented", Constructed(), Surface(trailing: ExtraSlot)),
|
||||||
|
("over-consent", Constructed(), Surface(trailing: new[] { ExtraSlot, ExtraSlot })),
|
||||||
|
("nested", Constructed(), Surface(extra: new Extra(Composition: true), trailing: ExtraSlot)),
|
||||||
|
("clean", Constructed(), Surface()),
|
||||||
|
}, snapshot =>
|
||||||
|
{
|
||||||
|
var entry = snapshot.SchemaCatalog.Single();
|
||||||
|
var last = entry.Members.OrderBy(member => member.Slot).Last();
|
||||||
|
entry.Members = entry.Members.Append(new CultSchemaMemberCatalogEntry { Slot = 7, MemberName = "Wider", TypeName = last.TypeName }).ToArray();
|
||||||
|
entry.SchemaId = WideId;
|
||||||
|
foreach (var record in snapshot.Records) record.SchemaId = WideId;
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The outcomes of both slot stores as data; SlotReadOnlyTests asserts every row from the committed files.
|
||||||
|
[Test]
|
||||||
|
public void SlotOutcomesVectorMatches() => Check("slot-outcomes.msgpack", SlotOutcomesFile());
|
||||||
|
|
||||||
// The value names every arm shape the vectors exist to hold other runtimes to.
|
// The value names every arm shape the vectors exist to hold other runtimes to.
|
||||||
[Test]
|
[Test]
|
||||||
public void ConstructedValueExercisesEveryArmShape()
|
public void ConstructedValueExercisesEveryArmShape()
|
||||||
|
|
@ -475,7 +615,7 @@ namespace GameCult.Caching.Tests
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
Assert.That(outcome, Is.EqualTo("read-only"), file);
|
Assert.That(outcome, Is.EqualTo("read-only"), file);
|
||||||
// The record is read whole; what makes it read-only is the slot rule of the slot-tolerance cut, not asserted here.
|
// The extra slot is read past; that the record then loads read-only is asserted in SlotReadOnlyTests, store by store.
|
||||||
Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past");
|
Assert.That(Describe(Decode(payload)), Is.EqualTo(golden), $"{file}: the extra slot is read past");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -781,7 +921,8 @@ namespace GameCult.Caching.Tests
|
||||||
Assert.That(File.ReadAllBytes(path), Is.EqualTo(new byte[] { 0x91 }), $"switch {off ?? "unset"} rewrites nothing");
|
Assert.That(File.ReadAllBytes(path), Is.EqualTo(new byte[] { 0x91 }), $"switch {off ?? "unset"} rewrites nothing");
|
||||||
File.Delete(path);
|
File.Delete(path);
|
||||||
|
|
||||||
foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles })
|
foreach (var body in new Action[] { UnionSurfaceVectorMatches, UnionRefusalVectorsRefused, BoundsOutcomesFromCommittedFiles,
|
||||||
|
ExtraSlotStoreVectorHoldsAMarkedACleanAndAUnitArmRecord, ExtraSlotConsentVectorHoldsTheConsentCases, SlotOutcomesVectorMatches })
|
||||||
body();
|
body();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,43 +18,16 @@ namespace GameCult.Caching.Tests
|
||||||
// compares them byte for byte. Other runtimes are held to the same text and the same verdicts.
|
// compares them byte for byte. Other runtimes are held to the same text and the same verdicts.
|
||||||
public class UniqueIndexVectorTests
|
public class UniqueIndexVectorTests
|
||||||
{
|
{
|
||||||
private const string WriteVariable = "CULTCACHE_WRITE_VECTORS";
|
private static readonly DateTimeOffset StoredAt = VectorFiles.StoredAt;
|
||||||
private static readonly DateTimeOffset StoredAt = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
private static bool Writing => VectorFiles.Writing;
|
||||||
private static bool Writing => Environment.GetEnvironmentVariable(WriteVariable) == "1";
|
|
||||||
|
|
||||||
private static string VectorDirectory()
|
private static string VectorDirectory() => VectorFiles.Directory("unique-index");
|
||||||
{
|
|
||||||
for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent)
|
|
||||||
{
|
|
||||||
var vectors = Path.Combine(dir.FullName, "tests", "vectors");
|
|
||||||
if (Directory.Exists(vectors)) return Path.Combine(vectors, "unique-index");
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new DirectoryNotFoundException($"tests/vectors not found above {TestContext.CurrentContext.TestDirectory}.");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string VectorPath(string name) => Path.Combine(VectorDirectory(), name);
|
private static string VectorPath(string name) => Path.Combine(VectorDirectory(), name);
|
||||||
|
|
||||||
private static void Check(string name, byte[] reference)
|
private static void Check(string name, byte[] reference) => VectorFiles.Check("unique-index", name, reference);
|
||||||
{
|
|
||||||
var path = VectorPath(name);
|
|
||||||
if (Writing)
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
|
||||||
File.WriteAllBytes(path, reference);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Assert.That(File.Exists(path), Is.True, $"vector {name} is committed");
|
private static byte[] Committed(string name) => VectorFiles.Committed("unique-index", name);
|
||||||
Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static byte[] Committed(string name)
|
|
||||||
{
|
|
||||||
var path = VectorPath(name);
|
|
||||||
Assert.That(File.Exists(path), Is.True, $"vector {name} is committed");
|
|
||||||
return File.ReadAllBytes(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- values.msgpack: [label, kind, value, text-or-nil] ----
|
// ---- values.msgpack: [label, kind, value, text-or-nil] ----
|
||||||
|
|
||||||
|
|
@ -290,5 +263,359 @@ namespace GameCult.Caching.Tests
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- refusals.msgpack: [label, ops, refusal]; the pair a refusal names is a function of the resulting state ----
|
||||||
|
|
||||||
|
private sealed record Schema(string Name, Type Role, Type Index, string SecondField);
|
||||||
|
|
||||||
|
private static Schema Plain(string suffix, string name) => new(
|
||||||
|
name,
|
||||||
|
EmittedDocumentTypes.Emit("UqVec" + suffix + "Role", name, name + ".v1", RoleFields),
|
||||||
|
EmittedDocumentTypes.Emit("UqVec" + suffix + "Index", name, name + ".v1", IndexFields),
|
||||||
|
"Category");
|
||||||
|
|
||||||
|
private static readonly Schema[] Schemas =
|
||||||
|
{
|
||||||
|
new("tests.unique_item", ItemRole, ItemIndex, "Category"),
|
||||||
|
new("tests.uq_pair",
|
||||||
|
EmittedDocumentTypes.Emit("UqVecPairRole", "tests.uq_pair", "tests.uq_pair.v1",
|
||||||
|
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "Alt") }),
|
||||||
|
EmittedDocumentTypes.Emit("UqVecPairIndex", "tests.uq_pair", "tests.uq_pair.v1",
|
||||||
|
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "Alt") }),
|
||||||
|
"Alt"),
|
||||||
|
new("tests.uq_alias",
|
||||||
|
EmittedDocumentTypes.Emit("UqVecAliasRole", "tests.uq_alias", "tests.uq_alias.v1",
|
||||||
|
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "Sku"), new("Alt", typeof(string), 1, Role: "alt") }),
|
||||||
|
EmittedDocumentTypes.Emit("UqVecAliasIndex", "tests.uq_alias", "tests.uq_alias.v1",
|
||||||
|
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "Sku"), new("Alt", typeof(string), 1, Index: "alt") }),
|
||||||
|
"Alt"),
|
||||||
|
new("tests.uq_wide",
|
||||||
|
EmittedDocumentTypes.Emit("UqVecWideRole", "tests.uq_wide", "tests.uq_wide.v1",
|
||||||
|
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Role: "\uFF21"), new("Alt", typeof(string), 1, Role: "\U00010400") }),
|
||||||
|
EmittedDocumentTypes.Emit("UqVecWideIndex", "tests.uq_wide", "tests.uq_wide.v1",
|
||||||
|
new EmittedDocumentTypes.Field[] { new("Sku", typeof(string), 0, Index: "\uFF21"), new("Alt", typeof(string), 1, Index: "\U00010400") }),
|
||||||
|
"Alt"),
|
||||||
|
Plain("Alpha", "tests.uq_alpha"),
|
||||||
|
Plain("Beta", "tests.uq_beta"),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static Schema SchemaNamed(string name) => Schemas.Single(schema => schema.Name == name);
|
||||||
|
|
||||||
|
private sealed record Rec(string Key, string Schema, string Sku, string Second = "");
|
||||||
|
|
||||||
|
private sealed record Op(string Kind, string? File = null, Rec[]? Records = null);
|
||||||
|
|
||||||
|
private sealed record Refusal(string K1, string K2, string Schema, string Alias);
|
||||||
|
|
||||||
|
private sealed record Case(string Label, Op[] Ops, Refusal? Refusal);
|
||||||
|
|
||||||
|
private const string Item = "tests.unique_item";
|
||||||
|
private const string Pair = "tests.uq_pair";
|
||||||
|
private const string Alias = "tests.uq_alias";
|
||||||
|
private const string Wide = "tests.uq_wide";
|
||||||
|
private const string Alpha = "tests.uq_alpha";
|
||||||
|
private const string Beta = "tests.uq_beta";
|
||||||
|
|
||||||
|
private static Rec[] Items_(params (string Key, string Sku)[] rows) => rows.Select(row => new Rec(row.Key, Item, row.Sku)).ToArray();
|
||||||
|
private static Op Open(string? file = null) => new("open", file);
|
||||||
|
private static Op Pull(string file) => new("pull", file);
|
||||||
|
private static Op Commit(params Rec[] records) => new("commit", Records: records);
|
||||||
|
|
||||||
|
private static readonly Rec[] PairRecords = { new("item:a", Pair, "S", "X"), new("item:b", Pair, "S", "Y"), new("item:c", Pair, "T", "Y") };
|
||||||
|
private static readonly Rec[] SurrogateRecords = Items_(("k\uFF01", "S"), ("k\U0001F600", "S"), ("k\uE000", "S"));
|
||||||
|
private static readonly Rec[] TypeRecords =
|
||||||
|
{
|
||||||
|
new("b:1", Alpha, "P"), new("a:1", Beta, "P"), new("b:2", Alpha, "P"), new("a:2", Beta, "P")
|
||||||
|
};
|
||||||
|
private static readonly Rec[] AliasRecords = { new("item:a", Alias, "S", "X"), new("item:b", Alias, "S", "Y"), new("item:c", Alias, "T", "Y") };
|
||||||
|
private static readonly Rec[] RepullBefore = Items_(("item:a1", "A"), ("item:b1", "C"));
|
||||||
|
private static readonly Rec[] RepullAfter = Items_(("item:a1", "A"), ("item:b1", "B"), ("item:b2", "B"), ("item:c1", "A"));
|
||||||
|
private static readonly Rec[] KeyCaseRecords = Items_(("a", "S"), ("z", "S"), ("B", "T"), ("C", "T"));
|
||||||
|
private static readonly Rec[] HolderCaseRecords = Items_(("a", "S"), ("A", "S"));
|
||||||
|
private static readonly Rec[] CrossSchemaRecords = { new("x:1", Alpha, "P"), new("y:1", Beta, "P") };
|
||||||
|
// Distinct under any trim, case-fold, NFC or NFKC of the value: 'S' and 's', 'S' and 'S ', NFC and NFD, fullwidth S.
|
||||||
|
private static readonly Rec[] DistinctValueRecords = Items_(
|
||||||
|
("k:1", "S"), ("k:2", "s"), ("k:3", "S "), ("k:4", "\u00E9"), ("k:5", "e\u0301"), ("k:6", "\uFF33"), ("k:7", " S"));
|
||||||
|
private static readonly Rec[] NormalKeyRecords = Items_(("\u00E9", "S"), ("e\u0301", "S"));
|
||||||
|
// Three holders of one value: code-unit order is e+U+0302, f, U+00E9; NFC would order f, U+00E9, e+U+0302 and NFD U+00E9, e+U+0302, f.
|
||||||
|
private static readonly Rec[] ThreeHolderRecords = Items_(("e\u0302", "S"), ("f", "S"), ("\u00E9", "S"));
|
||||||
|
// Both records violate both aliases: by code unit U+10400 (D801 DC00) precedes U+FF21, by code point U+FF21 does.
|
||||||
|
private static readonly Rec[] WideRecords = { new("item:a", Wide, "S", "X"), new("item:b", Wide, "S", "X") };
|
||||||
|
private static readonly Rec[] KeyPunctuationRecords = Items_(("_", "S"), ("a", "S"), ("Z", "T"), ("b", "T"));
|
||||||
|
private static readonly Rec[] SurrogateGroupRecords = Items_(("\uE000", "S"), ("\uE000b", "S"), ("\U0001F600", "T"), ("\U0001F600b", "T"));
|
||||||
|
|
||||||
|
private static readonly (string File, Rec[] Records)[] RefusalStores =
|
||||||
|
{
|
||||||
|
("pair-store.cc", PairRecords), ("surrogate-store.cc", SurrogateRecords), ("types-store.cc", TypeRecords),
|
||||||
|
("repull-before.cc", RepullBefore), ("repull-after.cc", RepullAfter), ("alias-store.cc", AliasRecords),
|
||||||
|
("key-case-store.cc", KeyCaseRecords), ("key-punctuation-store.cc", KeyPunctuationRecords),
|
||||||
|
("surrogate-groups-store.cc", SurrogateGroupRecords),
|
||||||
|
("holder-case-store.cc", HolderCaseRecords),
|
||||||
|
("cross-schema-store.cc", CrossSchemaRecords), ("distinct-values-store.cc", DistinctValueRecords),
|
||||||
|
("normal-keys-store.cc", NormalKeyRecords), ("normal-three-store.cc", ThreeHolderRecords), ("wide-store.cc", WideRecords),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static readonly Refusal SurrogateRefusal = new("k\U0001F600", "k\uE000", Item, "Sku");
|
||||||
|
private static readonly Refusal TypesRefusal = new("a:1", "a:2", Beta, "Sku");
|
||||||
|
private static readonly Refusal PairRefusal = new("item:b", "item:c", Pair, "Alt");
|
||||||
|
private static readonly Refusal AliasRefusal = new("item:a", "item:b", Alias, "Sku");
|
||||||
|
private static readonly Refusal RepullRefusal = new("item:a1", "item:c1", Item, "Sku");
|
||||||
|
private static readonly Refusal KeyCaseRefusal = new("B", "C", Item, "Sku");
|
||||||
|
private static readonly Refusal HolderCaseRefusal = new("A", "a", Item, "Sku");
|
||||||
|
private static readonly Refusal NormalKeyRefusal = new("e\u0301", "\u00E9", Item, "Sku");
|
||||||
|
private static readonly Refusal ThreeHolderRefusal = new("e\u0302", "f", Item, "Sku");
|
||||||
|
private static readonly Refusal WideRefusal = new("item:a", "item:b", Wide, "\U00010400");
|
||||||
|
private static readonly Refusal KeyPunctuationRefusal = new("Z", "b", Item, "Sku");
|
||||||
|
private static readonly Refusal SurrogateGroupRefusal = new("\U0001F600", "\U0001F600b", Item, "Sku");
|
||||||
|
|
||||||
|
private static readonly Case[] Cases =
|
||||||
|
{
|
||||||
|
new("batch-later-value-staged-first", new[] { Open(), Commit(Items_(("x", "A"), ("y", "A"), ("a", "B"), ("b", "B"))) },
|
||||||
|
new Refusal("a", "b", Item, "Sku")),
|
||||||
|
new("batch-surrogate-keys", new[] { Open(), Commit(SurrogateRecords) }, SurrogateRefusal),
|
||||||
|
new("batch-interleaved-types", new[] { Open(), Commit(TypeRecords) }, TypesRefusal),
|
||||||
|
new("batch-two-aliases", new[] { Open(), Commit(PairRecords) }, PairRefusal),
|
||||||
|
new("held-then-landing", new[] { Open(), Commit(Items_(("m", "S"))), Commit(Items_(("a", "S"))) },
|
||||||
|
new Refusal("a", "m", Item, "Sku")),
|
||||||
|
new("held-in-the-least-group", new[] { Open(), Commit(Items_(("m", "A"))), Commit(Items_(("z", "B"), ("y", "B"), ("a", "A"))) },
|
||||||
|
new Refusal("a", "m", Item, "Sku")),
|
||||||
|
new("swap-off-a-held-value", new[] { Open(), Commit(Items_(("a1", "A"))), Commit(Items_(("a1", "B"), ("c1", "A"))) }, null),
|
||||||
|
new("load-two-aliases", new[] { Open("pair-store.cc") }, PairRefusal),
|
||||||
|
new("load-surrogate-keys", new[] { Open("surrogate-store.cc") }, SurrogateRefusal),
|
||||||
|
new("load-interleaved-types", new[] { Open("types-store.cc") }, TypesRefusal),
|
||||||
|
new("repull-held-pair", new[] { Open("repull-before.cc"), Pull("repull-after.cc") }, RepullRefusal),
|
||||||
|
new("load-repull-after", new[] { Open("repull-after.cc") }, RepullRefusal),
|
||||||
|
new("commit-onto-a-loaded-holder", new[] { Open("repull-before.cc"), Commit(Items_(("item:z", "A"))) },
|
||||||
|
new Refusal("item:a1", "item:z", Item, "Sku")),
|
||||||
|
// Rows 14-19 extend the spec's floor of 13: orderings a code-point, case-insensitive or culture compare would flip.
|
||||||
|
new("batch-key-case", new[] { Open(), Commit(KeyCaseRecords) }, KeyCaseRefusal),
|
||||||
|
new("batch-key-punctuation", new[] { Open(), Commit(KeyPunctuationRecords) }, KeyPunctuationRefusal),
|
||||||
|
new("batch-key-case-in-a-group", new[] { Open(), Commit(Items_(("a", "S"), ("B", "S"))) },
|
||||||
|
new Refusal("B", "a", Item, "Sku")),
|
||||||
|
new("batch-surrogate-groups", new[] { Open(), Commit(SurrogateGroupRecords) }, SurrogateGroupRefusal),
|
||||||
|
new("batch-alias-case", new[] { Open(), Commit(AliasRecords) }, AliasRefusal),
|
||||||
|
new("load-alias-case", new[] { Open("alias-store.cc") }, AliasRefusal),
|
||||||
|
// Rows 20-22 load rows 14, 15 and 17's records, so a runtime with no batch commit still walks cross-group key order.
|
||||||
|
new("load-key-case", new[] { Open("key-case-store.cc") }, KeyCaseRefusal),
|
||||||
|
new("load-key-punctuation", new[] { Open("key-punctuation-store.cc") }, KeyPunctuationRefusal),
|
||||||
|
new("load-surrogate-groups", new[] { Open("surrogate-groups-store.cc") }, SurrogateGroupRefusal),
|
||||||
|
// Rows 23-24: two holders of one value that differ only by case are two holders; a case-folding dedupe would see one.
|
||||||
|
new("batch-holders-by-case", new[] { Open(), Commit(HolderCaseRecords) }, HolderCaseRefusal),
|
||||||
|
new("load-holders-by-case", new[] { Open("holder-case-store.cc") }, HolderCaseRefusal),
|
||||||
|
// Row 25: the held `a` is replaced only by the exact key `a`, never by `A`.
|
||||||
|
new("batch-replaced-by-case", new[] { Open(), Commit(Items_(("a", "S"))), Commit(Items_(("A", "S"))) }, HolderCaseRefusal),
|
||||||
|
// Rows 26-31: identity is exact. One value in two schemas is two groups; values differing in case, trailing or leading space, normalisation form or width are two values; keys differing only by normalisation form are two holders.
|
||||||
|
new("batch-value-in-two-schemas", new[] { Open(), Commit(CrossSchemaRecords) }, null),
|
||||||
|
new("load-value-in-two-schemas", new[] { Open("cross-schema-store.cc") }, null),
|
||||||
|
new("batch-distinct-values", new[] { Open(), Commit(DistinctValueRecords) }, null),
|
||||||
|
new("load-distinct-values", new[] { Open("distinct-values-store.cc") }, null),
|
||||||
|
new("batch-keys-by-normal-form", new[] { Open(), Commit(NormalKeyRecords) }, NormalKeyRefusal),
|
||||||
|
new("load-keys-by-normal-form", new[] { Open("normal-keys-store.cc") }, NormalKeyRefusal),
|
||||||
|
// Rows 32-33: one landing record sharing two unique aliases with a held one; the least alias by code unit is named ('Alt' before 'Sku'; 'Sku' before 'alt').
|
||||||
|
new("put-onto-a-held-pair", new[] { Open(), Commit(new Rec("m", Pair, "S", "X")), Commit(new Rec("a", Pair, "S", "X")) },
|
||||||
|
new Refusal("a", "m", Pair, "Alt")),
|
||||||
|
new("put-onto-a-held-alias-pair", new[] { Open(), Commit(new Rec("m", Alias, "S", "X")), Commit(new Rec("a", Alias, "S", "X")) },
|
||||||
|
new Refusal("a", "m", Alias, "Sku")),
|
||||||
|
// Rows 34-35: three holders of one value; the two least by code unit are named, which no NFC, NFD or culture sort of the keys yields.
|
||||||
|
new("batch-three-holders-normal-form", new[] { Open(), Commit(ThreeHolderRecords) }, ThreeHolderRefusal),
|
||||||
|
new("load-three-holders-normal-form", new[] { Open("normal-three-store.cc") }, ThreeHolderRefusal),
|
||||||
|
// Rows 36-37: two records violating two aliases, U+FF21 and U+10400; aliases compare by code unit, so U+10400 is named.
|
||||||
|
new("batch-alias-code-unit", new[] { Open(), Commit(WideRecords) }, WideRefusal),
|
||||||
|
new("load-alias-code-unit", new[] { Open("wide-store.cc") }, WideRefusal),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static byte[] RefusalStoreBytes(Rec[] records)
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxr-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.Combine(directory, "store.cc");
|
||||||
|
var roles = records.Select(record => SchemaNamed(record.Schema).Role).Distinct().ToArray();
|
||||||
|
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions
|
||||||
|
{
|
||||||
|
Registry = CultDocumentRegistry.ForTypes(roles), StoreFlushOnDispose = true
|
||||||
|
}))
|
||||||
|
{
|
||||||
|
cache.Clock = () => StoredAt;
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
var schema = SchemaNamed(record.Schema);
|
||||||
|
cache.UpsertAsync(schema.Role, EmittedDocumentTypes.New(schema.Role, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key))
|
||||||
|
.GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
cache.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
return File.ReadAllBytes(path);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] RefusalsFile()
|
||||||
|
{
|
||||||
|
var buffer = new ArrayBufferWriter<byte>();
|
||||||
|
var writer = new MessagePackWriter(buffer);
|
||||||
|
writer.WriteArrayHeader(Cases.Length);
|
||||||
|
foreach (var row in Cases)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(3);
|
||||||
|
writer.Write(row.Label);
|
||||||
|
writer.WriteArrayHeader(row.Ops.Length);
|
||||||
|
foreach (var op in row.Ops)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(2);
|
||||||
|
writer.Write(op.Kind);
|
||||||
|
if (op.Records != null)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(op.Records.Length);
|
||||||
|
foreach (var record in op.Records)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(4);
|
||||||
|
writer.Write(record.Key);
|
||||||
|
writer.Write(record.Schema);
|
||||||
|
writer.Write(record.Sku);
|
||||||
|
writer.Write(record.Second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (op.File == null) writer.WriteNil();
|
||||||
|
else writer.Write(op.File);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (row.Refusal == null) writer.WriteNil();
|
||||||
|
else
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(4);
|
||||||
|
writer.Write(row.Refusal.K1);
|
||||||
|
writer.Write(row.Refusal.K2);
|
||||||
|
writer.Write(row.Refusal.Schema);
|
||||||
|
writer.Write(row.Refusal.Alias);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.Flush();
|
||||||
|
return buffer.WrittenSpan.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Performs one op of a row against the open cache; the cache's own file is the only state it leaves behind.
|
||||||
|
private static void Perform(ref CultCache? cache, string path, Op op)
|
||||||
|
{
|
||||||
|
var indexTypes = Schemas.Select(schema => schema.Index).ToArray();
|
||||||
|
switch (op.Kind)
|
||||||
|
{
|
||||||
|
case "open":
|
||||||
|
if (op.File != null) File.WriteAllBytes(path, Committed(op.File));
|
||||||
|
cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = CultDocumentRegistry.ForTypes(indexTypes) });
|
||||||
|
break;
|
||||||
|
case "pull":
|
||||||
|
File.WriteAllBytes(path, Committed(op.File!));
|
||||||
|
cache!.PullAllBackingStoresAsync().GetAwaiter().GetResult();
|
||||||
|
break;
|
||||||
|
case "commit":
|
||||||
|
cache!.Commit(batch =>
|
||||||
|
{
|
||||||
|
foreach (var record in op.Records!)
|
||||||
|
{
|
||||||
|
var schema = SchemaNamed(record.Schema);
|
||||||
|
batch.Upsert(schema.Index, EmittedDocumentTypes.New(schema.Index, ("Sku", record.Sku), (schema.SecondField, record.Second)), new CultRecordKey(record.Key));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new InvalidOperationException(op.Kind);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void RefusalsHoldTheRule()
|
||||||
|
{
|
||||||
|
Check("refusals.msgpack", RefusalsFile());
|
||||||
|
foreach (var (file, records) in RefusalStores)
|
||||||
|
Check(file, RefusalStoreBytes(records));
|
||||||
|
|
||||||
|
// The committed file is what is walked, not the in-memory table.
|
||||||
|
var reader = new MessagePackReader(Committed("refusals.msgpack"));
|
||||||
|
var count = reader.ReadArrayHeader();
|
||||||
|
Assert.That(count, Is.EqualTo(Cases.Length));
|
||||||
|
var walked = 0;
|
||||||
|
var failed = new List<string>();
|
||||||
|
for (var i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(3));
|
||||||
|
var label = reader.ReadString()!;
|
||||||
|
var ops = new List<Op>();
|
||||||
|
for (var n = reader.ReadArrayHeader(); n > 0; n--)
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(2));
|
||||||
|
var kind = reader.ReadString()!;
|
||||||
|
if (kind == "commit")
|
||||||
|
{
|
||||||
|
var records = new List<Rec>();
|
||||||
|
for (var m = reader.ReadArrayHeader(); m > 0; m--)
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4));
|
||||||
|
records.Add(new Rec(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!));
|
||||||
|
}
|
||||||
|
|
||||||
|
ops.Add(new Op(kind, Records: records.ToArray()));
|
||||||
|
}
|
||||||
|
else ops.Add(new Op(kind, NextText(ref reader)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Refusal? refusal = null;
|
||||||
|
if (!reader.TryReadNil())
|
||||||
|
{
|
||||||
|
Assert.That(reader.ReadArrayHeader(), Is.EqualTo(4));
|
||||||
|
refusal = new Refusal(reader.ReadString()!, reader.ReadString()!, reader.ReadString()!, reader.ReadString()!);
|
||||||
|
}
|
||||||
|
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-uidxf-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
CultCache? cache = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.Combine(directory, "store.cc");
|
||||||
|
foreach (var op in ops.Take(ops.Count - 1))
|
||||||
|
Perform(ref cache, path, op);
|
||||||
|
|
||||||
|
var last = ops[^1];
|
||||||
|
if (refusal == null)
|
||||||
|
{
|
||||||
|
Assert.DoesNotThrow(() => Perform(ref cache, path, last), label);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
cache?.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
var before = last.Kind == "open" && last.File != null ? Committed(last.File)
|
||||||
|
: last.Kind == "pull" ? Committed(last.File!)
|
||||||
|
: File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty<byte>();
|
||||||
|
CultCache? attempt = cache;
|
||||||
|
var refused = Assert.Throws<InvalidOperationException>(() => Perform(ref attempt, path, last), label)!;
|
||||||
|
Assert.That(refused.Message, Is.EqualTo(
|
||||||
|
$"Records {refusal.K1} and {refusal.K2} ({refusal.Schema}) both hold unique index '{refusal.Alias}'; an index value names one record."), label);
|
||||||
|
if (last.Kind != "open") cache?.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
Assert.That(File.Exists(path) ? File.ReadAllBytes(path) : Array.Empty<byte>(), Is.EqualTo(before), $"{label}: the file is untouched");
|
||||||
|
}
|
||||||
|
|
||||||
|
walked++;
|
||||||
|
}
|
||||||
|
catch (AssertionException failure)
|
||||||
|
{
|
||||||
|
failed.Add($"{label}: {failure.Message}");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
cache?.Dispose();
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(failed, Is.Empty, "rows that failed");
|
||||||
|
Assert.That(walked, Is.EqualTo(37));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -513,7 +513,7 @@ namespace GameCult.Caching.Tests
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void AnOverrideOfASlotTheTypeNeverHadIsIgnoredOnLoadAsAPlainRecordIgnoresAnExtraSlot()
|
public void AnOverrideOfASlotTheTypeNeverHadIsReadPastOnLoadAsAPlainRecordReadsPastAnExtraSlot()
|
||||||
{
|
{
|
||||||
var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base",
|
var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base",
|
||||||
CultVariantOverride.Set(99, new byte[] { 0x01 }),
|
CultVariantOverride.Set(99, new byte[] { 0x01 }),
|
||||||
|
|
@ -1039,7 +1039,7 @@ namespace GameCult.Caching.Tests
|
||||||
}
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndStaysStoredUntilTheVariantIsWritten()
|
public void AnOverrideOfASlotTheTypeNeverHadIsReportedOnEveryLoadAndHoldsTheVariantReadOnly()
|
||||||
{
|
{
|
||||||
var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base",
|
var path = WriteStore(PlainGear("hand-base"), OverridingRecord("hand-variant", "hand-base",
|
||||||
CultVariantOverride.Set(99, new byte[] { 0x01 }),
|
CultVariantOverride.Set(99, new byte[] { 0x01 }),
|
||||||
|
|
@ -1059,14 +1059,22 @@ namespace GameCult.Caching.Tests
|
||||||
{
|
{
|
||||||
Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.EqualTo(new[] { 99 }));
|
Assert.That(reopened.BackingStores[0].LastSchemaMigrationReports.SelectMany(report => report.IgnoredExtraSlots), Is.EqualTo(new[] { 99 }));
|
||||||
Assert.That(reopened.Get<VariantGear>(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant"));
|
Assert.That(reopened.Get<VariantGear>(new CultRecordKey("hand-variant"))!.Name, Is.EqualTo("hand variant"));
|
||||||
reopened.Commit(batch => batch.UpsertVariant(new CultRecordKey("hand-variant"), new CultRecordKey("hand-base"), new[]
|
var variant = new CultRecordKey("hand-variant");
|
||||||
|
Assert.That(reopened.IsWritable(variant), Is.False, "an override of a slot the type lacks, which the catalog did not report, is a mark nothing explains");
|
||||||
|
var overrides = new[]
|
||||||
{
|
{
|
||||||
reopened.Override<VariantGear>(nameof(VariantGear.Name), "hand variant"),
|
reopened.Override<VariantGear>(nameof(VariantGear.Name), "hand variant"),
|
||||||
reopened.Override<VariantGear>(nameof(VariantGear.Code), "hand-code")
|
reopened.Override<VariantGear>(nameof(VariantGear.Code), "hand-code")
|
||||||
}));
|
};
|
||||||
|
var rewrite = Assert.Throws<CultSchemaConflictException>(() => reopened.Commit(batch => batch.UpsertVariant(variant, new CultRecordKey("hand-base"), overrides)))!;
|
||||||
|
Assert.That(rewrite.UnreadSlots, Is.Not.Null);
|
||||||
|
Assert.That(rewrite.UnreadSlots!.TypeName, Is.EqualTo(nameof(VariantGear)));
|
||||||
|
Assert.That(rewrite.UnreadSlots.SlotCount, Is.EqualTo(100));
|
||||||
|
Assert.Throws<CultSchemaConflictException>(() => reopened.Commit(batch => batch.Flatten(variant)));
|
||||||
|
Assert.Throws<CultSchemaConflictException>(() => reopened.Commit(batch => batch.Remove(variant)));
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2 }), "a write of the variant stores only what it was given");
|
Assert.That(OverriddenSlotsOn(path, "hand-variant").OrderBy(slot => slot), Is.EqualTo(new[] { 0, 2, 99 }), "the refused writes left the variant as stored");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int[] OverriddenSlotsOn(string path, string key) =>
|
private static int[] OverriddenSlotsOn(string path, string key) =>
|
||||||
|
|
|
||||||
50
tests/GameCult.Caching.Tests/VectorFiles.cs
Normal file
50
tests/GameCult.Caching.Tests/VectorFiles.cs
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.Tests
|
||||||
|
{
|
||||||
|
// The one place a C#-written vector directory is found, written and compared. The C# reference is the only writer
|
||||||
|
// (CULTCACHE_WRITE_VECTORS=1, never set in CI); every other run reads the committed bytes and compares them byte for byte.
|
||||||
|
internal static class VectorFiles
|
||||||
|
{
|
||||||
|
private const string WriteVariable = "CULTCACHE_WRITE_VECTORS";
|
||||||
|
|
||||||
|
internal static readonly DateTimeOffset StoredAt = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
||||||
|
|
||||||
|
internal static bool Writing => Environment.GetEnvironmentVariable(WriteVariable) == "1";
|
||||||
|
|
||||||
|
internal static string Directory(string vector)
|
||||||
|
{
|
||||||
|
for (var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); dir != null; dir = dir.Parent)
|
||||||
|
{
|
||||||
|
var vectors = System.IO.Path.Combine(dir.FullName, "tests", "vectors");
|
||||||
|
if (System.IO.Directory.Exists(vectors)) return System.IO.Path.Combine(vectors, vector);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new DirectoryNotFoundException($"tests/vectors not found above {TestContext.CurrentContext.TestDirectory}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static void Check(string vector, string name, byte[] reference)
|
||||||
|
{
|
||||||
|
var path = System.IO.Path.Combine(Directory(vector), name);
|
||||||
|
if (Writing)
|
||||||
|
{
|
||||||
|
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!);
|
||||||
|
File.WriteAllBytes(path, reference);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.That(File.Exists(path), Is.True, $"vector {name} is committed");
|
||||||
|
Assert.That(Convert.ToHexString(reference), Is.EqualTo(Convert.ToHexString(File.ReadAllBytes(path))), $"vector {name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static byte[] Committed(string vector, string name)
|
||||||
|
{
|
||||||
|
var path = System.IO.Path.Combine(Directory(vector), name);
|
||||||
|
Assert.That(File.Exists(path), Is.True, $"vector {name} is committed");
|
||||||
|
return File.ReadAllBytes(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
61
tests/GameCult.Mesh.Tests/BytesAfterPathTests.cs
Normal file
61
tests/GameCult.Mesh.Tests/BytesAfterPathTests.cs
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
using System;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Reflection;
|
||||||
|
using FluentAssertions;
|
||||||
|
using GameCult.Caching;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
using GameCult.Networking;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace GameCult.Mesh.Tests;
|
||||||
|
|
||||||
|
// The mesh's typed reads of a raw record decode the payload through the one-value rule: a tail is refused, never ignored.
|
||||||
|
public class BytesAfterPathTests
|
||||||
|
{
|
||||||
|
private const string Clause = "bytes after its first MessagePack value";
|
||||||
|
|
||||||
|
private static CultNetRawDocumentRecord WithTail(CultMeshCdnArtifactManifest manifest)
|
||||||
|
{
|
||||||
|
var payload = CultDocumentMessagePackSerialization.SerializeUntyped(manifest, typeof(CultMeshCdnArtifactManifest)).Concat(new byte[] { 0xc0 }).ToArray();
|
||||||
|
return new CultNetRawDocumentRecord
|
||||||
|
{
|
||||||
|
SchemaId = CultDocumentRegistry.Shared.GetRequired<CultMeshCdnArtifactManifest>().SchemaId,
|
||||||
|
RecordKey = "k",
|
||||||
|
StoredAt = "1970-01-01T00:00:00.000Z",
|
||||||
|
Payload = payload,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CultMeshCdnArtifactManifest Manifest() => new() { ArtifactId = "a", Version = "1", ContentHash = "h" };
|
||||||
|
|
||||||
|
private static string Refusal(Action read)
|
||||||
|
{
|
||||||
|
var thrown = Assert.Catch(read)!;
|
||||||
|
return thrown is TargetInvocationException { InnerException: { } inner } ? inner.ToString() : thrown.ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ASnapshotDecodeThroughABindingRefusesATail()
|
||||||
|
{
|
||||||
|
var registry = CultMeshCdn.CreateDocumentRegistry();
|
||||||
|
var snapshot = new CultNetSnapshotResponseRawMessage { Documents = new[] { WithTail(Manifest()) } };
|
||||||
|
Refusal(() => CultMesh.DecodeSnapshotDocuments<CultMeshCdnArtifactManifest>(snapshot, registry)).Should().Contain(Clause);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ASnapshotDecodeWithNoBindingRefusesATail()
|
||||||
|
{
|
||||||
|
var snapshot = new CultNetSnapshotResponseRawMessage { Documents = new[] { WithTail(Manifest()) } };
|
||||||
|
Refusal(() => CultMesh.DecodeSnapshotDocuments<CultMeshCdnArtifactManifest>(snapshot, new CultNetDocumentRegistry())).Should().Contain(Clause);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void ATypedReadOfOneSnapshotRecordRefusesATail()
|
||||||
|
{
|
||||||
|
var record = WithTail(Manifest());
|
||||||
|
var snapshot = new CultNetSnapshotResponseRawMessage { MessageId = "m", Documents = new[] { record } };
|
||||||
|
var read = typeof(CultMesh).GetMethod("ReadDocumentFromSnapshotResponse", BindingFlags.NonPublic | BindingFlags.Static)!
|
||||||
|
.MakeGenericMethod(typeof(CultMeshCdnArtifactManifest));
|
||||||
|
Refusal(() => read.Invoke(null, new object[] { snapshot, record.SchemaId, record.RecordKey })).Should().Contain(Clause);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -462,12 +462,12 @@ public sealed class CultMeshContentVectorTests
|
||||||
private static byte[] Reencode(CultMeshContentChunkResponseMessage message) =>
|
private static byte[] Reencode(CultMeshContentChunkResponseMessage message) =>
|
||||||
CultNetSchemaMessageSerialization.Serialize(message);
|
CultNetSchemaMessageSerialization.Serialize(message);
|
||||||
|
|
||||||
/// <summary>The one reader of manifest bytes: the document reader production stores and reads manifests with (CultPayloadDepth 256 included).</summary>
|
/// <summary>The one reader of manifest bytes: the production CultNet binding of the manifest (CultPayloadDepth 256 and the one-value rule included).</summary>
|
||||||
private static CultMeshCdnArtifactManifest ReadManifest(byte[] bytes) =>
|
private static CultMeshCdnArtifactManifest ReadManifest(byte[] bytes) =>
|
||||||
(CultMeshCdnArtifactManifest)CultDocumentMessagePackSerialization.DeserializeUntyped(typeof(CultMeshCdnArtifactManifest), bytes);
|
(CultMeshCdnArtifactManifest)CultMeshCdn.CreateDocumentRegistry().GetByDocumentType(typeof(CultMeshCdnArtifactManifest))!.PayloadDeserializer(bytes);
|
||||||
|
|
||||||
private static byte[] Reencode(CultMeshCdnArtifactManifest manifest) =>
|
private static byte[] Reencode(CultMeshCdnArtifactManifest manifest) =>
|
||||||
CultDocumentMessagePackSerialization.SerializeUntyped(manifest, typeof(CultMeshCdnArtifactManifest));
|
CultMeshCdn.CreateDocumentRegistry().GetByDocumentType(typeof(CultMeshCdnArtifactManifest))!.PayloadSerializer(manifest);
|
||||||
|
|
||||||
private static CultMeshContentChunkRequestMessage Request(string id, string hash, string key, int size) => new()
|
private static CultMeshContentChunkRequestMessage Request(string id, string hash, string key, int size) => new()
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
using GameCult.Networking;
|
using GameCult.Networking;
|
||||||
using MessagePack;
|
using MessagePack;
|
||||||
using NUnit.Framework;
|
using NUnit.Framework;
|
||||||
|
|
@ -168,6 +169,32 @@ public sealed class CultMeshProviderSessionWireTests
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void DecodeCommandDocumentRefusesBytesAfterItsFirstValue()
|
||||||
|
{
|
||||||
|
var command = new CultMeshProviderCommandWire
|
||||||
|
{
|
||||||
|
CommandId = "cmd-7", CommandKind = "pilot.thrust", ProviderId = "aetheria", ServiceInstanceId = "aetheria-42",
|
||||||
|
Payload = new Dictionary<string, object> { ["thrust"] = 1 }
|
||||||
|
};
|
||||||
|
var payload = Convert.FromBase64String(CultMeshProviderSessionWire.EncodePayload(command));
|
||||||
|
CultNetRawDocumentRecord Record(byte[] bytes) => new()
|
||||||
|
{
|
||||||
|
SchemaId = CultMeshProviderSessionWireContract.CommandSchema,
|
||||||
|
RecordKey = "provider-command:aetheria:aetheria-42:cmd-7",
|
||||||
|
PayloadEncoding = "messagepack",
|
||||||
|
Payload = bytes
|
||||||
|
};
|
||||||
|
|
||||||
|
Assert.That(CultMeshProviderSessionWire.DecodeCommandDocument(Record(payload)).CommandId, Is.EqualTo("cmd-7"));
|
||||||
|
var thrown = Assert.Throws<MessagePackSerializationException>(() =>
|
||||||
|
CultMeshProviderSessionWire.DecodeCommandDocument(Record(payload.Concat(new byte[] { 0xc0, 0xc0 }).ToArray())))!;
|
||||||
|
Assert.That(thrown.Message, Does.Contain("2 bytes after its first MessagePack value"));
|
||||||
|
var one = Assert.Throws<MessagePackSerializationException>(() =>
|
||||||
|
CultMeshProviderSessionWire.DecodeCommandDocument(Record(payload.Concat(new byte[] { 0xc0 }).ToArray())))!;
|
||||||
|
Assert.That(one.Message, Does.Contain("1 bytes after its first MessagePack value"));
|
||||||
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void LeaseResponse_RoundTrips_WithApplicationAcceptance()
|
public void LeaseResponse_RoundTrips_WithApplicationAcceptance()
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,8 @@ namespace GameCult.Mesh.Tests;
|
||||||
public sealed class CultMeshSingleFileReadabilityTests
|
public sealed class CultMeshSingleFileReadabilityTests
|
||||||
{
|
{
|
||||||
private const int CSharp = 1;
|
private const int CSharp = 1;
|
||||||
|
// The row `@directory` has no bytes: the walker makes an empty directory at the store path, and the write must refuse it untouched.
|
||||||
|
private const string DirectoryRow = "@directory";
|
||||||
private string _root = null!;
|
private string _root = null!;
|
||||||
|
|
||||||
[SetUp]
|
[SetUp]
|
||||||
|
|
@ -67,8 +69,15 @@ public sealed class CultMeshSingleFileReadabilityTests
|
||||||
foreach (var (name, write) in writes)
|
foreach (var (name, write) in writes)
|
||||||
{
|
{
|
||||||
var path = Path.Combine(_root, name.Replace(' ', '-') + ".cc");
|
var path = Path.Combine(_root, name.Replace(' ', '-') + ".cc");
|
||||||
var bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
|
var isDirectory = vector == DirectoryRow;
|
||||||
File.WriteAllBytes(path, bytes);
|
var bytes = Array.Empty<byte>();
|
||||||
|
if (isDirectory)
|
||||||
|
Directory.CreateDirectory(path);
|
||||||
|
else
|
||||||
|
{
|
||||||
|
bytes = File.ReadAllBytes(Path.Combine(VectorRoot(), vector));
|
||||||
|
File.WriteAllBytes(path, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
if (reads)
|
if (reads)
|
||||||
{
|
{
|
||||||
|
|
@ -96,8 +105,18 @@ public sealed class CultMeshSingleFileReadabilityTests
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.That(() => write(path), Throws.TypeOf<CultStoreUnreadableException>(), name);
|
// A directory at the path is refused as IOException ("not a store file"), not as an unreadable store.
|
||||||
File.ReadAllBytes(path).Should().Equal(bytes, name + " rewrote a file this runtime cannot read");
|
if (isDirectory)
|
||||||
|
Assert.That(() => write(path), Throws.TypeOf<IOException>(), name);
|
||||||
|
else
|
||||||
|
Assert.That(() => write(path), Throws.TypeOf<CultStoreUnreadableException>(), name);
|
||||||
|
if (isDirectory)
|
||||||
|
{
|
||||||
|
Directory.Exists(path).Should().BeTrue(name + " replaced a directory");
|
||||||
|
Directory.EnumerateFileSystemEntries(path).Should().BeEmpty(name + " wrote into a directory");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
File.ReadAllBytes(path).Should().Equal(bytes, name + " rewrote a file this runtime cannot read");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -115,7 +134,7 @@ public sealed class CultMeshSingleFileReadabilityTests
|
||||||
[Test]
|
[Test]
|
||||||
public void EveryVectorInTheFolderHasAManifestRow()
|
public void EveryVectorInTheFolderHasAManifestRow()
|
||||||
{
|
{
|
||||||
var rows = Vectors().Select(row => (string)row.Arguments[0]).Where(name => !name.StartsWith("..", StringComparison.Ordinal)).OrderBy(name => name, StringComparer.Ordinal);
|
var rows = Vectors().Select(row => (string)row.Arguments[0]).Where(name => !name.StartsWith("..", StringComparison.Ordinal) && name != DirectoryRow).OrderBy(name => name, StringComparer.Ordinal);
|
||||||
var files = Directory.GetFiles(VectorRoot(), "*.bin").Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal);
|
var files = Directory.GetFiles(VectorRoot(), "*.bin").Select(Path.GetFileName).OrderBy(name => name, StringComparer.Ordinal);
|
||||||
rows.Should().Equal(files);
|
rows.Should().Equal(files);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue