Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 75188fdefa | |||
| c88b939e76 | |||
| c77d734da5 | |||
| afab2d5a12 | |||
| 50f919c75b | |||
| 395d64e115 | |||
| 70f21da06a | |||
| 3d333e8466 | |||
| f9c72a2fd2 | |||
| 64f4d29147 | |||
| c229b86fde | |||
| 8edb2be2f4 | |||
| 6cfd22c786 | |||
| 568a60fa3e | |||
| c4a270bc7f | |||
| 694d2ced10 | |||
| d7e94648ac | |||
| d6708631da |
114 changed files with 9213 additions and 401 deletions
2
.github/workflows/cultmesh-portability.yml
vendored
2
.github/workflows/cultmesh-portability.yml
vendored
|
|
@ -158,7 +158,7 @@ jobs:
|
||||||
# (only the interop example built, inside the C# store tests). The target directory is under RUNNER_TEMP, away from the checkout.
|
# (only the interop example built, inside the C# store tests). The target directory is under RUNNER_TEMP, away from the checkout.
|
||||||
- name: Verify the CultCache document owner with the Rust cache
|
- name: Verify the CultCache document owner with the Rust cache
|
||||||
shell: bash
|
shell: bash
|
||||||
run: CARGO_TARGET_DIR="$RUNNER_TEMP/cultcache-rs-target" cargo test --manifest-path packages/cultcache-rs/Cargo.toml
|
run: CARGO_TARGET_DIR="$RUNNER_TEMP/cultcache-rs-target" cargo test --locked --manifest-path packages/cultcache-rs/Cargo.toml
|
||||||
|
|
||||||
# tests/GameCult.Caching.Tests, the C# reference cache's own suite (about 1,260 tests, under a minute), ran in no
|
# tests/GameCult.Caching.Tests, the C# reference cache's own suite (about 1,260 tests, under a minute), ran in no
|
||||||
# forge workflow; only the Mesh tests above exercised the reference.
|
# forge workflow; only the Mesh tests above exercised the reference.
|
||||||
|
|
|
||||||
4
.github/workflows/cultnet-interop.yml
vendored
4
.github/workflows/cultnet-interop.yml
vendored
|
|
@ -135,10 +135,10 @@ jobs:
|
||||||
python -m unittest discover -s packages/cultmesh-py/tests
|
python -m unittest discover -s packages/cultmesh-py/tests
|
||||||
|
|
||||||
- name: Verify Rust RUDP reliability
|
- name: Verify Rust RUDP reliability
|
||||||
run: cargo test --manifest-path packages/cultnet-rs/Cargo.toml rudp
|
run: cargo test --locked --manifest-path packages/cultnet-rs/Cargo.toml rudp
|
||||||
|
|
||||||
- name: Verify Rust CultMesh runtime-local surface
|
- name: Verify Rust CultMesh runtime-local surface
|
||||||
run: cargo test --manifest-path packages/cultmesh-rs/Cargo.toml
|
run: cargo test --locked --manifest-path packages/cultmesh-rs/Cargo.toml
|
||||||
|
|
||||||
- name: Verify C# RUDP reliability
|
- name: Verify C# RUDP reliability
|
||||||
run: dotnet test tests/GameCult.Networking.Tests/GameCult.Networking.Tests.csproj --filter "FullyQualifiedName~Rudp"
|
run: dotnet test tests/GameCult.Networking.Tests/GameCult.Networking.Tests.csproj --filter "FullyQualifiedName~Rudp"
|
||||||
|
|
|
||||||
|
|
@ -212,8 +212,8 @@ identity-owned C# session after provider route replacement.
|
||||||
Rust package tests:
|
Rust package tests:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
cargo test --manifest-path packages/cultcache-rs/Cargo.toml
|
cargo test --locked --manifest-path packages/cultcache-rs/Cargo.toml
|
||||||
cargo test --manifest-path packages/cultnet-rs/Cargo.toml
|
cargo test --locked --manifest-path packages/cultnet-rs/Cargo.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
Kotlin package build:
|
Kotlin package build:
|
||||||
|
|
|
||||||
|
|
@ -115,7 +115,7 @@
|
||||||
"validPutPayloadHex": "81A16101",
|
"validPutPayloadHex": "81A16101",
|
||||||
"whiteSpaceCodePoints": ["0009","000A","000B","000C","000D","0020","0085","00A0","1680","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","200A","2028","2029","202F","205F","3000"],
|
"whiteSpaceCodePoints": ["0009","000A","000B","000C","000D","0020","0085","00A0","1680","2000","2001","2002","2003","2004","2005","2006","2007","2008","2009","200A","2028","2029","202F","205F","3000"],
|
||||||
"notWhiteSpaceCodePoints": ["200B","200C","200D","2060","FEFF","180E","001C","001D","001E","001F"],
|
"notWhiteSpaceCodePoints": ["200B","200C","200D","2060","FEFF","180E","001C","001D","001E","001F"],
|
||||||
"codePointCases": "For every code point in whiteSpaceCodePoints, each of schemaId, recordKey, storedAt, sourceRuntimeId, sourceAgentId, sourceRole and one tags entry, set alone to a string of just that code point (and to that code point repeated three times), is refused; for every code point in notWhiteSpaceCodePoints the same string is not blank and the put is admitted.",
|
"codePointCases": "For every code point in whiteSpaceCodePoints, each of schemaId, recordKey, storedAt, sourceRuntimeId, sourceAgentId, sourceRole and one tags entry, set alone to a string of just that code point (and to that code point repeated three times), is refused; for every code point in notWhiteSpaceCodePoints the same string in each of those fields except storedAt is not blank and the put is admitted (a storedAt must be a stamp: a non-blank storedAt that is no stamp is refused message_invalid, field document, after the write authority, and is covered by tests/vectors/stored-at-stamp).",
|
||||||
"cases": [
|
"cases": [
|
||||||
{ "name": "valid_document_is_admitted", "set": {}, "outcome": "admitted" },
|
{ "name": "valid_document_is_admitted", "set": {}, "outcome": "admitted" },
|
||||||
{ "name": "empty_schema_id", "set": { "schemaId": "" }, "outcome": "refused" },
|
{ "name": "empty_schema_id", "set": { "schemaId": "" }, "outcome": "refused" },
|
||||||
|
|
@ -140,7 +140,7 @@
|
||||||
{ "name": "zero_width_space_schema_id_is_a_name", "set": { "schemaId": "\u200b" }, "outcome": "admitted" },
|
{ "name": "zero_width_space_schema_id_is_a_name", "set": { "schemaId": "\u200b" }, "outcome": "admitted" },
|
||||||
{ "name": "zero_width_space_record_key_is_a_name", "set": { "recordKey": "\u200b" }, "outcome": "admitted" },
|
{ "name": "zero_width_space_record_key_is_a_name", "set": { "recordKey": "\u200b" }, "outcome": "admitted" },
|
||||||
{ "name": "zero_width_space_tag_is_a_name", "set": { "tags": ["\u200b"] }, "outcome": "admitted" },
|
{ "name": "zero_width_space_tag_is_a_name", "set": { "tags": ["\u200b"] }, "outcome": "admitted" },
|
||||||
{ "name": "bom_stored_at_is_a_name", "set": { "storedAt": "\ufeff" }, "outcome": "admitted" },
|
{ "name": "bom_record_key_is_a_name", "set": { "recordKey": "\ufeff" }, "outcome": "admitted" },
|
||||||
{ "name": "mongolian_vowel_separator_source_role_is_a_name", "set": { "sourceRole": "\u180e" }, "outcome": "admitted" },
|
{ "name": "mongolian_vowel_separator_source_role_is_a_name", "set": { "sourceRole": "\u180e" }, "outcome": "admitted" },
|
||||||
{ "name": "padded_name_is_a_name", "set": { "schemaId": "\u00a0a\u2003", "tags": [" a"] }, "outcome": "admitted" },
|
{ "name": "padded_name_is_a_name", "set": { "schemaId": "\u00a0a\u2003", "tags": [" a"] }, "outcome": "admitted" },
|
||||||
{ "name": "file_separator_schema_id_is_a_name", "set": { "schemaId": "\u001c" }, "outcome": "admitted" },
|
{ "name": "file_separator_schema_id_is_a_name", "set": { "schemaId": "\u001c" }, "outcome": "admitted" },
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ A connection is receipt-capable when its `cultnet.hello.v0` (sent before its fir
|
||||||
- **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests.
|
- **What is remembered.** Per entry a server keeps only fixed-size digests (SHA-256) of the `messageId` and of the content hash, plus the answer (`admitted`, or the refusal's code and details). Memory per entry is bounded independent of id length and payload size. Lookup compares digests.
|
||||||
- **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table.
|
- **Remembered-write limits.** The per-session table holds at most 1024 entries. A server MAY also enforce a server-wide memory budget across all sessions (reference: 32 MiB of remembered entries). A NEW write (a key not already in the table) that would exceed either bound is refused `limit_reached` for that write alone, `messageId` the write's id, `details` all null; the write is not applied and the session continues. A resend of a remembered key is still answered from the table.
|
||||||
- **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session.
|
- **Limit refusals are remembered.** A `limit_reached` refusal is remembered like any other: a resend of the same id and content hash replays `limit_reached`. No slot is freed mid-session, so an id refused at a bound never becomes admitted later in that session (a server at the cap keeps the refused digest pair in a bounded refusal set of at most 1024 per session, outside the entry budget; past it, a further new write is refused `limit_reached` without being remembered, and the session refuses every id it has not seen for the rest of its life, even if a server-wide budget has since freed, so a refusal that was not remembered is never followed by an admission of the same id). The client mints a new id on a new session.
|
||||||
- **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case.
|
- **Required fields.** A put or delete whose document or `schemaId` is null or missing is refused `message_invalid` with `details.field` `"document"` or `"schemaId"` (the first missing in that order) and fixed text; the write is not applied. A put or delete that is present but names nothing is not "missing": the write is refused `message_invalid` exactly as a frame that does not read, with `details.field` `"message"`, the fixed unreadable text and the write's id (vector `error_v1_empty_schema_id_is_unreadable_and_names_the_write`), and is never applied; a server must not hand it to its store. Names nothing means, on a put or delete, a `schemaId` or `recordKey` that is blank; on a put's document, also a `storedAt` that is blank, a `payload` of zero bytes (a MessagePack value is at least one byte, so an empty payload is no document), a `sourceRuntimeId`, `sourceAgentId` or `sourceRole` that is present and blank, and a `tags` entry that is blank. **Blank** means empty or made only of Unicode White_Space code points (the PropList property) and nothing else, in every runtime, whatever its platform's own whitespace test says. The set is exactly U+0009 to U+000D, U+0020, U+0085, U+00A0, U+1680, U+2000 to U+200A, U+2028, U+2029, U+202F, U+205F and U+3000. U+200B, U+200C, U+200D, U+2060, U+FEFF, U+180E and U+001C to U+001F are not White_Space (some platforms' whitespace tests accept U+001C to U+001F or U+180E, for example Java's `Character.isWhitespace` and older .NET, so a runtime must check the list, not its platform test): a string of them names something. The vectors list both sets (`whiteSpaceCodePoints`, `notWhiteSpaceCodePoints`) and a runtime checks every code point in every covered field. A put's `storedAt` is the exception to "present and not blank is a name": it must be a stamp, so a non-blank `storedAt` that is no stamp (`tests/vectors/stored-at-stamp`) is refused `message_invalid` with `details.field` `"document"`, after the write authority has decided (an ungranted peer is told `write_not_authorized` first), and nothing is stored or minted; only a blank `storedAt` is refused as unreadable with field `"message"`, before the authority. The non-White_Space code point rows therefore name something in every covered field except `storedAt`. Absent or nil source ids and tags, and an empty `tags` list, are valid. The rule holds on every connection, receipt-capable or not (a legacy connection is refused with the `error.v0` of the same code, details and text); the vectors' `blankFields` section lists each case.
|
||||||
- **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`.
|
- **Fixed texts.** The `error` string of these refusals is fixed, so vectors can compare it: `message_invalid: messageId exceeds 256 bytes`, `message_invalid: document is required`, `message_invalid: schemaId is required`, `message_invalid: the message could not be read` (a frame that does not decode, or a write that names nothing; the text is the same for both and never carries a decoder's own wording), `message_invalid: receipt sessions require the reliable channel`, `limit_reached: the session's remembered-write limit is full`.
|
||||||
- **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. A repeat of a packet the server already took, or a packet too far ahead to be taken, draws no further answer (rule 1).
|
- **Reliable channel only.** A write is accepted on a receipt session only on a channel that is reliable and not sequenced (reliable ordered and reliable unordered both qualify; a sequenced channel drops superseded packets, so a write on it can vanish without trace). A write arriving on any other channel is refused `message_invalid` with `details.field: "channel"` (the transport profile's word for it) and fixed text. Receipts and refusals travel the reliable channel; a write that could be lost without trace cannot be answered once. The server refuses a sequenced write whenever it can read the write's id: from the delivered frame, and from a superseded packet that carries its whole frame. A write sent as several fragments whose frame is superseded is dropped whole and may go unanswered; the client's unconfirmed outcome covers it. A repeat of a packet the server already took, or a packet too far ahead to be taken, draws no further answer (rule 1).
|
||||||
- Past the session (or a server's longer retention, if it keeps one) a resend is a new write.
|
- Past the session (or a server's longer retention, if it keeps one) a resend is a new write.
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
CultLib measures its C# test suites with Stryker.NET, pinned as a repo-local
|
CultLib measures its C# test suites with Stryker.NET, pinned as a repo-local
|
||||||
tool in `dotnet-tools.json`. Coverage is added project by project; a project
|
tool in `dotnet-tools.json`. Coverage is added project by project; a project
|
||||||
with no `stryker-config.json` next to its test project has not been scoped
|
with no `stryker-config.json` next to its test project has not been scoped
|
||||||
yet (see "Follow-ups" below for what's outstanding).
|
yet (see "Scope and follow-ups" below for what's outstanding).
|
||||||
|
|
||||||
## Running it
|
## Running it
|
||||||
|
|
||||||
|
|
@ -126,14 +126,17 @@ project as precedent for the `vstest` caveat itself - that caveat is
|
||||||
`mtp`-runner-specific and documented above for `static readonly` field
|
`mtp`-runner-specific and documented above for `static readonly` field
|
||||||
initializers, and GameCult.Geometry has never actually hit it.
|
initializers, and GameCult.Geometry has never actually hit it.
|
||||||
|
|
||||||
## Follow-ups
|
## Scope and follow-ups
|
||||||
|
|
||||||
CultMath.Tests and GameCult.Geometry.Tests are the only projects currently
|
CultMath.Tests, GameCult.Geometry.Tests and GameCult.Caching.Tests are the
|
||||||
scoped. GameCult.Geometry's `stryker-config.json` has no `test-runner`
|
projects currently scoped. GameCult.Geometry's `stryker-config.json` has no
|
||||||
entry: the project is NUnit 4 with `NUnit3TestAdapter` and
|
`test-runner` entry: the project is NUnit 4 with `NUnit3TestAdapter` and
|
||||||
`Microsoft.NET.Test.Sdk`, so Stryker's default `vstest` runner applies
|
`Microsoft.NET.Test.Sdk`, so Stryker's default `vstest` runner applies
|
||||||
directly (no `mtp` workaround needed, unlike CultMath.Tests's xunit.v3
|
directly (no `mtp` workaround needed, unlike CultMath.Tests's xunit.v3
|
||||||
suite).
|
suite). GameCult.Caching.Tests is the same shape (NUnit, default `vstest`)
|
||||||
|
and its `stryker-config.json` is driven the same way, by the repo-local
|
||||||
|
`dotnet-stryker` tool: `dotnet tool restore` once, then `dotnet tool run
|
||||||
|
dotnet-stryker` from `tests/GameCult.Caching.Tests`.
|
||||||
|
|
||||||
GameCult.Geometry has only ever been run `--since:`-scoped to the cut that
|
GameCult.Geometry has only ever been run `--since:`-scoped to the cut that
|
||||||
added surface nets; a full-project baseline (running mutation against all of
|
added surface nets; a full-project baseline (running mutation against all of
|
||||||
|
|
@ -151,7 +154,83 @@ file, anything under `Fixtures/`) loses `--since` line-level scoping for that
|
||||||
run; a future cut whose diff is C#-only, against a base commit that already
|
run; a future cut whose diff is C#-only, against a base commit that already
|
||||||
has `stryker-config.json`, should scope correctly.
|
has `stryker-config.json`, should scope correctly.
|
||||||
|
|
||||||
`GameCult.Caching`, `GameCult.Mesh` (+ `.Quic`, `.Quic.Native`),
|
### GameCult.Caching.Tests
|
||||||
|
|
||||||
|
`tests/GameCult.Caching.Tests/stryker-config.json` mutates all of
|
||||||
|
`src/GameCult.Caching` (the project is `GameCult.Caching.csproj`; Standard
|
||||||
|
mutation level, `break` 0, the same thresholds as Geometry). Caching was
|
||||||
|
scoped before Networking because its suite is deterministic and quick (1415
|
||||||
|
tests, about 40 s) over about 7.8k lines of source, where GameCult.Networking
|
||||||
|
is about 20k lines and its tests lean on socket timing, so a mutant timing out
|
||||||
|
there says little about the test. A full Caching run is about 2,900 candidate
|
||||||
|
mutants (3,602 created, minus 730 that fail to compile) at 1.25 to 1.6 s
|
||||||
|
each, so about 1 to 1.3 hours; run it detached and read the JSON report,
|
||||||
|
never as one attached session.
|
||||||
|
|
||||||
|
The first measurement was a narrow pass over the three files that carry the
|
||||||
|
store-replace and element-id machinery, from `tests/GameCult.Caching.Tests`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet tool run dotnet-stryker \
|
||||||
|
-m '**/src/GameCult.Caching/StoreReplace.cs' \
|
||||||
|
-m '**/src/GameCult.Caching/CultStoreLinks.cs' \
|
||||||
|
-m '**/src/GameCult.Caching/CultElementIds.cs'
|
||||||
|
```
|
||||||
|
|
||||||
|
A clean single run of that command tests 243 to 265 mutants (the count moves with the revision) and scored 85.28% on
|
||||||
|
568a60fa, before any test of this scope was added; 86.79% on d38b5728, with
|
||||||
|
the first batch of tests; and 87.17% on 76d9e7d1, with the throwing-enumerator
|
||||||
|
case, in about 10 minutes. The first measurement of it overlapped two Stryker
|
||||||
|
runs, so its numbers (and the 114 survivors it showed for `CultElementIds.cs`,
|
||||||
|
where tests that kill a mutant by hand reported `Survived` or `Timeout`) are
|
||||||
|
not measurements and are not quoted here. Never run two Stryker runs against
|
||||||
|
one checkout at once, and check `ps` before starting one. A clean single run
|
||||||
|
on `CultElementIds.cs` alone, after the tests below, tested 228 mutants: 208
|
||||||
|
`Killed`, 8 `Timeout`, 12 `Survived`, a score of 94.74%. The 12th survivor
|
||||||
|
was 321, which the throwing-enumerator case in `ListsOfValuesHoldNoIds` now
|
||||||
|
kills; the other 11 are equivalent mutants. With that case a clean single run
|
||||||
|
on `CultElementIds.cs` is 209 `Killed`, 8 `Timeout`, 11 `Survived`, 95.18%.
|
||||||
|
|
||||||
|
Real gaps that run exposed, each now pinned by `ElementIdWalkTests` (and each
|
||||||
|
failing when its mutant is applied by hand): the order of the id path (the
|
||||||
|
codec refuses at the first type MessagePack cannot allow, so order is what an
|
||||||
|
error names first; `Append` to `Prepend` at 163, 167 and 180); the
|
||||||
|
union-cycle guard in `ConcreteTypes` (248; the mutant is a stack overflow, so
|
||||||
|
the red is a crashed test host: a depth guard would need production code in
|
||||||
|
`ConcreteTypes`, since the recursion is private and a stack overflow cannot be
|
||||||
|
caught by the test, so the regression test is left to crash loudly); a list of values reporting that it holds an
|
||||||
|
id (`Holds`, 322); `[Key]` read through an override (`KeyOf`, 583); and the
|
||||||
|
`IsWrittenWhole` and `IsKeyed` predicates (629 to 631 and 636: formatter,
|
||||||
|
`[MessagePackObject]` and `KeyAsPropertyName` inherited from a base, and the
|
||||||
|
`[DataContract]` without `[MessagePackObject]` and `||` arms).
|
||||||
|
|
||||||
|
Equivalent survivors in `CultElementIds.cs` (11), each hand-checked against the full
|
||||||
|
suite; none can be told apart by any test:
|
||||||
|
|
||||||
|
- 142, 251, 636 (second arm), 506 (`IsDefined(UnionAttribute, false)` to
|
||||||
|
`true`): `UnionAttribute` is `Inherited = false`, so its `inherit` argument
|
||||||
|
changes nothing. Likewise 630's `DataContractAttribute` `inherit` argument
|
||||||
|
(`DataContractAttribute` is `Inherited = false`).
|
||||||
|
- 264, 542: `group.First()` to `FirstOrDefault()` over `GroupBy` groups, which
|
||||||
|
are never empty.
|
||||||
|
- 491: `new StringBuilder(bytes.Length * 2)` to `/ 2` changes only the initial
|
||||||
|
capacity.
|
||||||
|
- 578: removing the `Nullable.GetUnderlyingType` unwrap in `WalksAsOwner`;
|
||||||
|
`Classify` unwraps nullables itself, so the unwrap is dead. A cut
|
||||||
|
candidate, not a test gap.
|
||||||
|
- 645, 657: `Append(type)` to `Prepend(type)` in `ElementTypeOf` and
|
||||||
|
`MapTypes`; `type` matches only when it is itself `IEnumerable<>`,
|
||||||
|
`IDictionary<,>` or `IReadOnlyDictionary<,>`, and none of those has a second
|
||||||
|
matching interface, so the position of `type` never decides the answer.
|
||||||
|
|
||||||
|
The Windows-only P/Invoke group is exercised only on Windows: its mutants are
|
||||||
|
`NoCoverage` on Linux and are a target for a Windows-leg mutation run, not a
|
||||||
|
Linux gap or a finding. `StoreReplace.cs` line 45 is a different case: its
|
||||||
|
negated `IsOSPlatform(Windows)` mutant is `Survived` on Linux, because the
|
||||||
|
negated branch calls `ReplaceWindows`, whose `DllNotFound` fallback gives the
|
||||||
|
same result.
|
||||||
|
|
||||||
|
`GameCult.Mesh` (+ `.Quic`, `.Quic.Native`),
|
||||||
`GameCult.Networking` (+ `.WebSockets`), and the non-.NET runtimes
|
`GameCult.Networking` (+ `.WebSockets`), and the non-.NET runtimes
|
||||||
(`cultcache-rs/py/ts`, `cultmesh-*`, `cultnet-*`) each still need their own
|
(`cultcache-rs/py/ts`, `cultmesh-*`, `cultnet-*`) each still need their own
|
||||||
`stryker-config.json` (or ecosystem-equivalent tool: `cargo-mutants` for
|
`stryker-config.json` (or ecosystem-equivalent tool: `cargo-mutants` for
|
||||||
|
|
|
||||||
|
|
@ -16,10 +16,10 @@ this page.
|
||||||
|
|
||||||
| | CultCache | CultMath | Geometry | CultNet | CultMesh |
|
| | CultCache | CultMath | Geometry | CultNet | CultMesh |
|
||||||
| --- | --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- | --- |
|
||||||
| **C#** | [first call, tested](#c-first-call) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [first call, tested](#c-cultnet-first-call); [package](../../src/GameCult.Networking) | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` |
|
| **C#** | [first call, tested](#c-first-call); [update, tested](#c-update-with-retry) | [package](../../packages/cultmath/README.md); snippet not yet | [package](../../src/GameCult.Geometry/README.md); snippet not yet | [first call, tested](#c-cultnet-first-call); [package](../../src/GameCult.Networking) | [quickstart](../../src/GameCult.Mesh/docs/durable-node-quickstart.md); run by `scripts/verify-eve-getting-started.ps1` |
|
||||||
| **TypeScript** | [first call, tested](#typescript-first-call) | not yet | not yet | [first call, tested](#typescript-cultnet-first-call); [package](../../packages/cultnet-ts/README.md) | [package](../../packages/cultmesh-ts/README.md); snippet not yet; [recipe: two-client .io game](io-game.md), tested with two live clients, browser page not yet run |
|
| **TypeScript** | [first call, tested](#typescript-first-call); [update, tested](#typescript-update-with-retry) | not yet | not yet | [first call, tested](#typescript-cultnet-first-call); [package](../../packages/cultnet-ts/README.md) | [package](../../packages/cultmesh-ts/README.md); snippet not yet; [recipe: two-client .io game](io-game.md), tested with two live clients, browser page not yet run |
|
||||||
| **Rust** | [first call, tested](#rust-first-call) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [first call, tested](#rust-cultnet-first-call); [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples) | [package](../../packages/cultmesh-rs); snippet not yet |
|
| **Rust** | [first call, tested](#rust-first-call); [update, tested](#rust-update-with-retry) | [native core](../../packages/cultmath/native/cultmath-core); snippet not yet | not yet | [first call, tested](#rust-cultnet-first-call); [package](../../packages/cultnet-rs/README.md), [examples](../../packages/cultnet-rs/examples) | [package](../../packages/cultmesh-rs); snippet not yet |
|
||||||
| **Python** | [first call, tested](#python-first-call) | not yet | not yet | [first call, tested](#python-cultnet-first-call); [package](../../packages/cultnet-py/README.md) | [package](../../packages/cultmesh-py/README.md); snippet not yet |
|
| **Python** | [first call, tested](#python-first-call); [update, tested](#python-update-with-retry) | not yet | not yet | [first call, tested](#python-cultnet-first-call); [package](../../packages/cultnet-py/README.md) | [package](../../packages/cultmesh-py/README.md); snippet not yet |
|
||||||
|
|
||||||
For the whole
|
For the whole
|
||||||
picture of what each piece is for, see the
|
picture of what each piece is for, see the
|
||||||
|
|
@ -70,6 +70,37 @@ public static class FirstCall
|
||||||
|
|
||||||
`FirstCall.Run` is the call; a second test in that file fails if the block above stops matching it. More: `src/GameCult.Caching/README.md`.
|
`FirstCall.Run` is the call; a second test in that file fails if the block above stops matching it. More: `src/GameCult.Caching/README.md`.
|
||||||
|
|
||||||
|
## C# update with retry
|
||||||
|
|
||||||
|
Change a record you may not be the only one changing. `Update` reads the record as its store holds it, hands your function a fresh
|
||||||
|
copy of it, and commits the result only if the record is still the one it read. If another writer got there first, it
|
||||||
|
reads again and runs your function again, up to five times. The source is
|
||||||
|
[`tests/GameCult.Caching.Tests/UpdateCallTests.cs`](../../tests/GameCult.Caching.Tests/UpdateCallTests.cs); the
|
||||||
|
snippet reuses `Item` from the first call.
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using GameCult.Caching;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
|
||||||
|
public static class UpdateCall
|
||||||
|
{
|
||||||
|
public static CultUpdateResult<Item> Restock(string path, CultRecordKey key)
|
||||||
|
{
|
||||||
|
var cache = CultCacheMessagePack.Create(path);
|
||||||
|
return cache.Update<Item>(key, item =>
|
||||||
|
{
|
||||||
|
item ??= new Item { Name = "Potion", Category = "Consumable" };
|
||||||
|
item.Value += 10;
|
||||||
|
return item;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The write is durable when `Update` returns, with no `PushAll`. A lost race re-runs the function, so keep it free of
|
||||||
|
side effects; when the attempts run out the result says `LostRace` instead of throwing. `UpsertAsync` plus
|
||||||
|
`PushAll` is the blind opt-out: last writer wins.
|
||||||
|
|
||||||
## TypeScript first call
|
## TypeScript first call
|
||||||
|
|
||||||
Same shape in TypeScript: the document is a name, a schema id and a `parse`-able schema (zod here),
|
Same shape in TypeScript: the document is a name, a schema id and a `parse`-able schema (zod here),
|
||||||
|
|
@ -119,12 +150,36 @@ async function firstCall(path: string) {
|
||||||
The test imports from `../src/index` where the block reads `@gamecult/cultcache-ts`; a second test in that file fails if the block stops matching the rest. More:
|
The test imports from `../src/index` where the block reads `@gamecult/cultcache-ts`; a second test in that file fails if the block stops matching the rest. More:
|
||||||
`packages/cultcache-ts/README.md`.
|
`packages/cultcache-ts/README.md`.
|
||||||
|
|
||||||
|
## TypeScript update with retry
|
||||||
|
|
||||||
|
`update` changes a record from what the store holds now and commits only if it is still that record. The source is the
|
||||||
|
second region of
|
||||||
|
[`packages/cultcache-ts/test/first-call.test.ts`](../../packages/cultcache-ts/test/first-call.test.ts), which uses the
|
||||||
|
definitions of the first call above; a test in that file fails if the block stops matching the file.
|
||||||
|
|
||||||
|
```ts
|
||||||
|
async function restock(path: string) {
|
||||||
|
const cache = CultCache.builder()
|
||||||
|
.withRegistry(registry)
|
||||||
|
.withGenericStore(new SingleFileMessagePackBackingStore(path))
|
||||||
|
.build();
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
return cache.update(itemDocument, "item:potion", (item) => {
|
||||||
|
const current = item ?? { name: "Potion", category: "Consumable", value: 0 };
|
||||||
|
return { ...current, value: current.value + 10 };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The write is durable when the promise resolves. A lost race re-runs the function, so keep it free of side effects; when the
|
||||||
|
attempts run out the result says `lostRace` instead of throwing. `put` is the blind opt-out: last writer wins.
|
||||||
|
|
||||||
## Rust first call
|
## Rust first call
|
||||||
|
|
||||||
Same shape with `derive(DatabaseEntry)`: slots are `#[cultcache(key = N)]`, so the `.cc` payload is the
|
Same shape with `derive(DatabaseEntry)`: slots are `#[cultcache(key = N)]`, so the `.cc` payload is the
|
||||||
slot-indexed array the C# runtime writes. The source is
|
slot-indexed array the C# runtime writes. The source is
|
||||||
[`packages/cultcache-rs/tests/first_call.rs`](../../packages/cultcache-rs/tests/first_call.rs), run by
|
[`packages/cultcache-rs/tests/first_call.rs`](../../packages/cultcache-rs/tests/first_call.rs), run by
|
||||||
`cargo test --test first_call` in `packages/cultcache-rs`; a second test in that file fails if the block
|
`cargo test --locked --test first_call` in `packages/cultcache-rs`; a second test in that file fails if the block
|
||||||
below stops matching the file. The Rust runtime has no name lookup, so the read-back is by key. More:
|
below stops matching the file. The Rust runtime has no name lookup, so the read-back is by key. More:
|
||||||
`packages/cultcache-rs/README.md`.
|
`packages/cultcache-rs/README.md`.
|
||||||
|
|
||||||
|
|
@ -165,6 +220,36 @@ fn first_call() -> anyhow::Result<()> {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Rust update with retry
|
||||||
|
|
||||||
|
`update` changes a record from what the store holds now and commits only if it is still that record. The source is the
|
||||||
|
second region of
|
||||||
|
[`packages/cultcache-rs/tests/first_call.rs`](../../packages/cultcache-rs/tests/first_call.rs), which uses the `Item` of
|
||||||
|
the first call above; a test in that file fails if the block stops matching the file.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn restock(path: &Path) -> anyhow::Result<CultUpdateResult<Item>> {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Item>()?;
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?;
|
||||||
|
cache.pull_all_backing_stores()?;
|
||||||
|
cache.update::<Item, _>("item:potion", |item| {
|
||||||
|
// The store's copy of the record, or nothing if it has none; this may run again on a lost race.
|
||||||
|
let mut item = item.unwrap_or(Item {
|
||||||
|
name: "Potion".to_string(),
|
||||||
|
category: "Consumable".to_string(),
|
||||||
|
value: 0,
|
||||||
|
});
|
||||||
|
item.value += 10;
|
||||||
|
Some(item)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The write is durable when `update` returns. A lost race re-runs the closure, so keep it free of side effects; when the
|
||||||
|
attempts run out (five, or the bound `update_with_attempts` takes) the outcome says `LostRace` instead of failing, and
|
||||||
|
returning `None` writes nothing and says `Declined`. `put` is the blind opt-out: last writer wins.
|
||||||
|
|
||||||
## Python first call
|
## Python first call
|
||||||
|
|
||||||
A document is a definition with an encoder and decoder; `name=` registers the name lookup, `indexes=` unique lookups and
|
A document is a definition with an encoder and decoder; `name=` registers the name lookup, `indexes=` unique lookups and
|
||||||
|
|
@ -220,6 +305,29 @@ def first_call() -> Item:
|
||||||
return reopened.get_required(item_doc, "item:potion")
|
return reopened.get_required(item_doc, "item:potion")
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Python update with retry
|
||||||
|
|
||||||
|
`update` changes a record from what the store holds now and commits only if it is still that record. The source is the
|
||||||
|
`update` region of
|
||||||
|
[`packages/cultcache-py/tests/test_first_call.py`](../../packages/cultcache-py/tests/test_first_call.py), which uses the
|
||||||
|
definitions of the first call above; a test in that file fails if the block stops matching the file.
|
||||||
|
|
||||||
|
```python
|
||||||
|
def restock(path: Path):
|
||||||
|
cache = (
|
||||||
|
CultCache.builder()
|
||||||
|
.register_document_type(item_doc)
|
||||||
|
.add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
return cache.update(item_doc, "item:potion", lambda item: Item("Potion", "Consumable", (item.value if item else 0) + 10))
|
||||||
|
```
|
||||||
|
|
||||||
|
The write is durable when `update` returns. A lost race re-runs the function, so keep it free of side effects; when the
|
||||||
|
attempts run out the result's `outcome` is `UpdateOutcome.LOST_RACE` instead of a raise. `put` is the blind opt-out: last
|
||||||
|
writer wins.
|
||||||
|
|
||||||
## CultNet first calls
|
## CultNet first calls
|
||||||
|
|
||||||
CultNet is schema-v0 messages in length-prefixed frames. The first call in each runtime is the same
|
CultNet is schema-v0 messages in length-prefixed frames. The first call in each runtime is the same
|
||||||
|
|
@ -341,7 +449,7 @@ async function firstCall() {
|
||||||
### Rust CultNet first call
|
### Rust CultNet first call
|
||||||
|
|
||||||
Source: [`packages/cultnet-rs/tests/first_call.rs`](../../packages/cultnet-rs/tests/first_call.rs), run by
|
Source: [`packages/cultnet-rs/tests/first_call.rs`](../../packages/cultnet-rs/tests/first_call.rs), run by
|
||||||
`cargo test --test first_call` in `packages/cultnet-rs`. More: `packages/cultnet-rs/README.md`.
|
`cargo test --locked --test first_call` in `packages/cultnet-rs`. More: `packages/cultnet-rs/README.md`.
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
use std::net::{TcpListener, TcpStream};
|
use std::net::{TcpListener, TcpStream};
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,19 @@
|
||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `parse_stored_at` (was `_parse_stored_at`) is public: cultmesh-py judges a peer's `storedAt` by it. A peer's `document_put_raw` whose `storedAt` is missing, empty or not a stamp is refused `message_invalid` (field `document`) and nothing is minted or stored (variants:ruling:raw-put-without-stamp-refused). A subscribed server's stampless put now raises `ValueError` from `CultMeshClient.read_next_change` instead of being stored under a minted stamp.
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- `CultCache.update` and `update_global` return `UpdateResult(outcome, document)`; `UpdateOutcome` is `UPDATED`, `DECLINED` or `LOST_RACE`. The single-file store gains `read(key)` (the record and its content version, the SHA-256 C# computes) and `commit_if(envelope, expected)`, the version is the SHA-256 of the record re-encoded as `[key, schemaId, storedAt, payload]` with minimal MessagePack encodings, which is the SHA-256 of the record's stored bytes exactly when the writer encoded it minimally (every writer in this library does; a record another writer encoded with a wider-than-needed length or integer has the version of its minimal form, not of its bytes). Both looked up by the cache and not `BackingStore` protocol members (variants:cut_spec:cut-update-with-retry-py).
|
||||||
- `read_payload_tree(type_name, payload, decoder=None)`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged.
|
- `read_payload_tree(type_name, payload, decoder=None)`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
- A write that replaces a record is stored at a later `stored_at` than the record it replaces, in the single-file store, the JSONL store and a store-less cache, with no exception for identical content (`variants:ruling:py-rs-update-follow-csharp-restamp`); the cache holds the stamp the store wrote. A stored_at is read only in the shape the runtimes write (an RFC 3339 date-time with T, a fraction of 1 to 9 digits, Z or a numeric offset), the same on every Python; any other text, or an instant outside `datetime`'s range, is not a stamp and a write over it is stamped at now. A write over a record stamped at the last instant `datetime` can hold (9999-12-31T23:59:59.999999Z) is refused with `ValueError` and writes nothing.
|
||||||
|
- `store_write_lock` looks for a link at the store path again once the lock is held, so a link put at the path while a writer waited on the lock is refused with `StoreLinkedError` and the lock released, instead of being replaced. `JsonLinesBackingStore` takes no cross-process lock (only its thread lock, under which it already refuses a link), so it has no lock wait to re-check across. Pinned by a test that puts the link while a foreign process holds the lock.
|
||||||
- `CultCache.is_writable` does not look at store paths: a link at a store path is not predicted, and the write refuses it with `StoreLinkedError`. Pinned by a test through a cache attached to a linked store.
|
- `CultCache.is_writable` does not look at store paths: a link at a store path is not predicted, and the write refuses it with `StoreLinkedError`. Pinned by a test through a cache attached to a linked store.
|
||||||
- `JsonLinesBackingStore` writes LF line endings on Windows. It wrote CRLF there, through `write_text` newline translation, before the commit_store_file change (#154, adbbb780).
|
- `JsonLinesBackingStore` writes LF line endings on Windows. It wrote CRLF there, through `write_text` newline translation, before the commit_store_file change (#154, adbbb780).
|
||||||
- `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import.
|
- `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import.
|
||||||
|
|
@ -16,7 +23,9 @@
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
- `push_all(envelopes, soft=False)` on the single-file and JSONL stores and on the `BackingStore` protocol takes the soft flag, as TypeScript, Rust and C# do: a soft push writes only where nothing is at the path; over a store the reader reads, empty included, it returns without writing, and over a file the reader cannot read it refuses (`StoreUnreadableError` for the single-file store). The `BackingStore` protocol's default `push_all` cannot tell an absent store from an empty one (a pull returns a list), so a soft push through it always skips a store whose pull returns; a store kind that can tell nothing-at-the-path overrides `push_all`. A store kind that overrides `push_all(envelopes)` without the keyword fails when a caller passes `soft`.
|
- `update(document, key, apply, *, max_attempts=5)` and `update_global(document, apply, *, max_attempts=5)` replace the blind pair. Each attempt reads the record from the home store, runs `apply` on a copy (None when absent) outside every lock, and commits only if the record is unchanged, running `apply` again on a lost race; it returns `UpdateResult`, not the value. `apply` returning None writes nothing; a lost race that outlasts `max_attempts` returns `LOST_RACE`; a store without `read` and `commit_if` (JSON lines among them) refuses before `apply`. `put` is the blind write.
|
||||||
|
- `push` returns the `stored_at` it wrote and `push_all` a dict of them by key (`{}` when a soft push skipped); a store returning None keeps the cache's own stamp.
|
||||||
|
- `push_all(envelopes, soft=False)` on the single-file and JSONL stores and on the `BackingStore` protocol takes the soft flag, as TypeScript, Rust and C# do: a soft push writes only where nothing is at the path; over a store the reader reads, empty included, it returns without writing, and over a file the reader cannot read it refuses (`StoreUnreadableError` for the single-file store). The `BackingStore` protocol has `pull_or_absent()`: the envelopes the reader reads, `None` when nothing is at the path, a raise when the path cannot be read; its default is `pull_all()`, which cannot say absent, so a store that inherits it is always read. The default `push_all` decides soft by it (writes where it says absent, skips over a store it read, refuses where it raises), and the single-file and JSONL stores expose their pull through it; an inheriting store that can tell nothing-at-the-path overrides `pull_or_absent`, not `push_all`. Before, a soft push through the default always skipped. A store kind that overrides `push_all(envelopes)` without the keyword fails when a caller passes `soft`.
|
||||||
- A write at a store path that is a link, live or dangling, is refused with the new `StoreLinkedError` (an `OSError`, exported from `cultcache_py`; `.path` names the path) before the lock and the replace, for the single-file store and for `JsonLinesBackingStore` (`push`, `push_all`, `delete`), and nothing is written. Before, a write at a live link replaced the link with a regular file, and at a dangling link both stores raised `FileNotFoundError`. A caller catching `FileNotFoundError` there now catches `StoreLinkedError` (`OSError`); configure the link's target as the store path. On Windows a junction or any reparse point counts as a link. Reads through a live link work.
|
- A write at a store path that is a link, live or dangling, is refused with the new `StoreLinkedError` (an `OSError`, exported from `cultcache_py`; `.path` names the path) before the lock and the replace, for the single-file store and for `JsonLinesBackingStore` (`push`, `push_all`, `delete`), and nothing is written. Before, a write at a live link replaced the link with a regular file, and at a dangling link both stores raised `FileNotFoundError`. A caller catching `FileNotFoundError` there now catches `StoreLinkedError` (`OSError`); configure the link's target as the store path. On Windows a junction or any reparse point counts as a link. Reads through a live link work.
|
||||||
- A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`.
|
- A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`.
|
||||||
- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` `<type> payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#.
|
- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` `<type> payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#.
|
||||||
|
|
|
||||||
|
|
@ -94,8 +94,7 @@ store.
|
||||||
- `put(...)`
|
- `put(...)`
|
||||||
- `put_envelopes(...)`
|
- `put_envelopes(...)`
|
||||||
- `put_global(...)`
|
- `put_global(...)`
|
||||||
- `update(...)`
|
- `update(document, key, apply, *, max_attempts=5)` and `update_global(document, apply, *, max_attempts=5)`: the conditional golden path; reads the record as its home store holds it now, runs `apply` on a fresh copy, writes only if the record is unchanged and runs `apply` again on a lost race; returns `UpdateResult(outcome, document)` with `UpdateOutcome.UPDATED`, `DECLINED` (`apply` returned `None`) or `LOST_RACE` (see below)
|
||||||
- `update_global(...)`
|
|
||||||
- `delete(...)`
|
- `delete(...)`
|
||||||
- `delete_global(...)`
|
- `delete_global(...)`
|
||||||
- `is_writable(document, key)` (advisory: would a write or removal of the key be refused; see below)
|
- `is_writable(document, key)` (advisory: would a write or removal of the key be refused; see below)
|
||||||
|
|
@ -143,6 +142,8 @@ used by the TypeScript, Rust, and C# runtimes:
|
||||||
canonicalSchemaJson, compatibleSchemaIds, members]`
|
canonicalSchemaJson, compatibleSchemaIds, members]`
|
||||||
- records: `[key, schemaId, storedAt, payload]`
|
- records: `[key, schemaId, storedAt, payload]`
|
||||||
|
|
||||||
|
`update` needs a home store with the optional `read(key)` (the record at the key, of any type, and its content version, from one read) and `commit_if(envelope, expected)` (write only if the record at the key is still that version, under the store's write lock). The content version is the lowercase hex SHA-256 of the record re-encoded as the store writes it, `[key, schemaId, storedAt, payload]` with minimal MessagePack encodings, the same value C# computes (`tests/vectors/content-version`). It equals the SHA-256 of the record's bytes in the file exactly when the writer encoded the record minimally, as every writer in this library does; a record encoded with a wider-than-needed length or integer has the version of its minimal form. The single-file store has both; `JsonLinesBackingStore` and a store without them refuse `update` before `apply` runs. A cache with no store copies its held instance. Every write that replaces a record is stored at a later `stored_at` than the one it replaces, in each store kind and in a store-less cache; `push` and `push_all` return the stamp written, and the cache holds that stamp.
|
||||||
|
|
||||||
Large corpora should use a sharded store or a real database. A single snapshot file is a scalpel, not a forklift.
|
Large corpora should use a sharded store or a real database. A single snapshot file is a scalpel, not a forklift.
|
||||||
|
|
||||||
## Store format and rollback
|
## Store format and rollback
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
from .backing_store import BackingStore, CultCacheEnvelope
|
from .backing_store import BackingStore, CultCacheEnvelope
|
||||||
from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember
|
from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember
|
||||||
from . import leaves
|
from . import leaves
|
||||||
from .cache import CultCache, CultCacheBuilder, ForeignRecord
|
from .cache import CultCache, CultCacheBuilder, ForeignRecord, UpdateOutcome, UpdateResult
|
||||||
from .documents import (
|
from .documents import (
|
||||||
MAX_PAYLOAD_DEPTH,
|
MAX_PAYLOAD_DEPTH,
|
||||||
MAX_SLOT_KEY,
|
MAX_SLOT_KEY,
|
||||||
|
|
@ -42,6 +42,8 @@ __all__ = [
|
||||||
"StoreUnreadableError",
|
"StoreUnreadableError",
|
||||||
"UnionArm",
|
"UnionArm",
|
||||||
"UnreadSlots",
|
"UnreadSlots",
|
||||||
|
"UpdateOutcome",
|
||||||
|
"UpdateResult",
|
||||||
"read_payload_tree",
|
"read_payload_tree",
|
||||||
"leaves",
|
"leaves",
|
||||||
"database_entry_field",
|
"database_entry_field",
|
||||||
|
|
|
||||||
|
|
@ -58,21 +58,35 @@ class CultCacheEnvelope:
|
||||||
|
|
||||||
|
|
||||||
class BackingStore(Protocol):
|
class BackingStore(Protocol):
|
||||||
|
"""The members every store has. A store may also offer `read(key) -> tuple[CultCacheEnvelope, str] | None` (the record at the
|
||||||
|
key, of any type, and its content version, from one read) and `commit_if(envelope, expected: str | None) -> str | None` (write
|
||||||
|
only if the record at the key is still the version `expected`, None meaning no record; returns the stored_at written, or None
|
||||||
|
with nothing written). They are looked up with getattr, not Protocol members, and `CultCache.update` needs both. `push` and
|
||||||
|
`push_all` may report the stored_at they wrote (`str` and `dict[str, str]` by key); None keeps the cache's own stamp. A store
|
||||||
|
stamps a record that replaces another strictly later than the one it replaces."""
|
||||||
|
|
||||||
def pull_all(self) -> list[CultCacheEnvelope]:
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
||||||
...
|
...
|
||||||
|
|
||||||
def push(self, envelope: CultCacheEnvelope) -> None:
|
def push(self, envelope: CultCacheEnvelope) -> str | None:
|
||||||
...
|
...
|
||||||
|
|
||||||
def delete(self, type: str, key: str) -> None:
|
def delete(self, type: str, key: str) -> None:
|
||||||
...
|
...
|
||||||
|
|
||||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None:
|
def pull_or_absent(self) -> list[CultCacheEnvelope] | None:
|
||||||
# A pull that returns is a store the reader read, empty included: the protocol cannot say "absent", so a soft push
|
"""The envelopes the reader reads (empty included), None when nothing is at the path; raises when the path cannot be read.
|
||||||
# through this default always skips. A store that can tell nothing-at-the-path overrides push_all.
|
|
||||||
pulled = self.pull_all()
|
The default is pull_all, which cannot say absent: a store that inherits it is always a store the reader read. A store kind
|
||||||
if soft:
|
that can tell nothing-at-the-path overrides this, and the default push_all then writes where nothing is on a soft push."""
|
||||||
|
return self.pull_all()
|
||||||
|
|
||||||
|
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> dict[str, str] | None:
|
||||||
|
# Soft: write only where pull_or_absent says nothing is; skip over a store it read, empty included; its raise refuses.
|
||||||
|
pulled = self.pull_or_absent()
|
||||||
|
if soft and pulled is not None:
|
||||||
return
|
return
|
||||||
|
pulled = pulled or []
|
||||||
existing = {(envelope.type, envelope.key): envelope for envelope in pulled}
|
existing = {(envelope.type, envelope.key): envelope for envelope in pulled}
|
||||||
for envelope in envelopes:
|
for envelope in envelopes:
|
||||||
existing[(envelope.type, envelope.key)] = envelope
|
existing[(envelope.type, envelope.key)] = envelope
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import enum
|
||||||
import functools
|
import functools
|
||||||
import math
|
import math
|
||||||
import threading
|
import threading
|
||||||
|
|
@ -9,13 +10,39 @@ from typing import Any, Callable, Generic, Iterable, TypeVar
|
||||||
|
|
||||||
from .backing_store import BackingStore, CultCacheEnvelope
|
from .backing_store import BackingStore, CultCacheEnvelope
|
||||||
from .documents import DocumentDefinition, UnreadSlots, extract_value
|
from .documents import DocumentDefinition, UnreadSlots, extract_value
|
||||||
from .stores import SchemaConflictError, retyped, written_owns
|
from .stores import SchemaConflictError, parse_stored_at, retyped, stored_at_over, written_owns
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
F = TypeVar("F", bound=Callable[..., Any])
|
F = TypeVar("F", bound=Callable[..., Any])
|
||||||
GLOBAL_KEY = "__global__"
|
GLOBAL_KEY = "__global__"
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateOutcome(enum.Enum):
|
||||||
|
UPDATED = "updated"
|
||||||
|
DECLINED = "declined"
|
||||||
|
LOST_RACE = "lost_race"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UpdateResult(Generic[T]):
|
||||||
|
"""What `CultCache.update` did and the cache's record at the key after it (None when it holds nothing). The document decides
|
||||||
|
nothing: it is a read after the outcome."""
|
||||||
|
|
||||||
|
outcome: UpdateOutcome
|
||||||
|
document: T | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _Expected:
|
||||||
|
"""What an update's commit requires to still hold. A cache with no store requires the held envelope object at the key (None:
|
||||||
|
nothing held); a store-backed one requires the type's home to be `store` and the record there to have content version
|
||||||
|
`version` (None: no record)."""
|
||||||
|
|
||||||
|
held: CultCacheEnvelope | None = None
|
||||||
|
store: BackingStore | None = None
|
||||||
|
version: str | None = None
|
||||||
|
|
||||||
|
|
||||||
def _locked(method: F) -> F:
|
def _locked(method: F) -> F:
|
||||||
"""Runs a read holding the cache's lock. Reads are reentrant and may run inside a mutation."""
|
"""Runs a read holding the cache's lock. Reads are reentrant and may run inside a mutation."""
|
||||||
|
|
||||||
|
|
@ -482,7 +509,7 @@ class CultCache:
|
||||||
def put(self, document: DocumentDefinition[T], key: str, value: T) -> None:
|
def put(self, document: DocumentDefinition[T], key: str, value: T) -> None:
|
||||||
self._put(document, key, value)
|
self._put(document, key, value)
|
||||||
|
|
||||||
def _put(self, document: DocumentDefinition[T], key: str, value: T) -> None:
|
def _put(self, document: DocumentDefinition[T], key: str, value: T, expected: _Expected | None = None) -> str:
|
||||||
self._assert_registered(document)
|
self._assert_registered(document)
|
||||||
if document.global_document and key != GLOBAL_KEY:
|
if document.global_document and key != GLOBAL_KEY:
|
||||||
raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}")
|
raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}")
|
||||||
|
|
@ -494,7 +521,7 @@ class CultCache:
|
||||||
schema_id=catalog_entry.schema_id,
|
schema_id=catalog_entry.schema_id,
|
||||||
catalog_entry=catalog_entry,
|
catalog_entry=catalog_entry,
|
||||||
)
|
)
|
||||||
self._write(document, [(envelope, value)], batch=False)
|
return self._write(document, [(envelope, value)], batch=False, expected=expected)
|
||||||
|
|
||||||
@_mutating
|
@_mutating
|
||||||
def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T:
|
def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T:
|
||||||
|
|
@ -562,9 +589,12 @@ class CultCache:
|
||||||
*,
|
*,
|
||||||
batch: bool,
|
batch: bool,
|
||||||
unread: dict[str, UnreadSlots | None] | None = None,
|
unread: dict[str, UnreadSlots | None] | None = None,
|
||||||
) -> None:
|
expected: _Expected | None = None,
|
||||||
|
) -> str:
|
||||||
"""Validates every record (home and name and index extractors included) before the store is
|
"""Validates every record (home and name and index extractors included) before the store is
|
||||||
touched; once the store accepts, applying to the cache has no fallible step."""
|
touched; once the store accepts, applying to the cache has no fallible step. With `expected` (an update's commit) the write
|
||||||
|
also requires the record to be unchanged since it was read and returns 'mismatch', having written and installed nothing,
|
||||||
|
when it is not; otherwise 'committed'. A record is held under the stored_at its store reports writing."""
|
||||||
type = document.type
|
type = document.type
|
||||||
# A batch naming a key twice lands the last record for it, as C# CultCacheBatch stages one operation per key. The
|
# A batch naming a key twice lands the last record for it, as C# CultCacheBatch stages one operation per key. The
|
||||||
# judge, the store and the apply all read this one list.
|
# judge, the store and the apply all read this one list.
|
||||||
|
|
@ -584,13 +614,34 @@ class CultCache:
|
||||||
)
|
)
|
||||||
store = self._store_for_type(type)
|
store = self._store_for_type(type)
|
||||||
legacy_key = self._state.legacy_global_keys.get(type)
|
legacy_key = self._state.legacy_global_keys.get(type)
|
||||||
if store is not None:
|
stamps: dict[str, Any] = {}
|
||||||
if batch:
|
if expected is not None:
|
||||||
store.push_all([envelope for envelope, _ in records])
|
if legacy_key is not None:
|
||||||
|
raise self._legacy_global_refusal(document, legacy_key)
|
||||||
|
if expected.store is None:
|
||||||
|
held = self._state.envelopes.get(type, {}).get(records[0][0].key)
|
||||||
|
if store is not None or held is not expected.held:
|
||||||
|
return "mismatch"
|
||||||
|
elif store is not expected.store:
|
||||||
|
return "mismatch"
|
||||||
else:
|
else:
|
||||||
store.push(records[0][0])
|
stamp = store.commit_if(records[0][0], expected.version) # type: ignore[attr-defined]
|
||||||
|
if stamp is None:
|
||||||
|
return "mismatch"
|
||||||
|
stamps[records[0][0].key] = stamp
|
||||||
|
elif store is not None:
|
||||||
|
if batch:
|
||||||
|
reported = store.push_all([envelope for envelope, _ in records])
|
||||||
|
stamps = reported if isinstance(reported, dict) else {}
|
||||||
|
else:
|
||||||
|
stamps[records[0][0].key] = store.push(records[0][0])
|
||||||
if legacy_key is not None:
|
if legacy_key is not None:
|
||||||
self._retire_legacy_global(store, type, legacy_key)
|
self._retire_legacy_global(store, type, legacy_key)
|
||||||
|
else:
|
||||||
|
for envelope, _ in records:
|
||||||
|
held_at = next((held for held in (by_type.get(envelope.key) for by_type in self._state.envelopes.values()) if held is not None), None)
|
||||||
|
if held_at is not None:
|
||||||
|
stamps[envelope.key] = stored_at_over(envelope.stored_at, held_at.stored_at)
|
||||||
self._state.legacy_global_keys.pop(type, None)
|
self._state.legacy_global_keys.pop(type, None)
|
||||||
values = self._state.values.setdefault(type, {})
|
values = self._state.values.setdefault(type, {})
|
||||||
envelopes = self._state.envelopes.setdefault(type, {})
|
envelopes = self._state.envelopes.setdefault(type, {})
|
||||||
|
|
@ -598,6 +649,9 @@ class CultCache:
|
||||||
for other, key in displaced:
|
for other, key in displaced:
|
||||||
self._drop(other, key)
|
self._drop(other, key)
|
||||||
for envelope, value, keys in admitted:
|
for envelope, value, keys in admitted:
|
||||||
|
stamp = stamps.get(envelope.key)
|
||||||
|
if isinstance(stamp, str) and parse_stored_at(stamp) is not None:
|
||||||
|
envelope = replace(envelope, stored_at=stamp)
|
||||||
self._remove_lookups(type, envelope.key)
|
self._remove_lookups(type, envelope.key)
|
||||||
values[envelope.key] = value
|
values[envelope.key] = value
|
||||||
envelopes[envelope.key] = envelope
|
envelopes[envelope.key] = envelope
|
||||||
|
|
@ -609,6 +663,7 @@ class CultCache:
|
||||||
self._state.names[(type, keys.name)] = envelope.key
|
self._state.names[(type, keys.name)] = envelope.key
|
||||||
for index, index_value in keys.indexes.items():
|
for index, index_value in keys.indexes.items():
|
||||||
self._state.indexes[(type, index, index_value)] = envelope.key
|
self._state.indexes[(type, index, index_value)] = envelope.key
|
||||||
|
return "committed"
|
||||||
|
|
||||||
def _refuse_retype(self, document: DocumentDefinition[Any], written: list[CultCacheEnvelope]) -> list[tuple[str, str]]:
|
def _refuse_retype(self, document: DocumentDefinition[Any], written: list[CultCacheEnvelope]) -> list[tuple[str, str]]:
|
||||||
"""The one key-identity judge (variants:ruling:cache-refuses-retype): a key holds one record whatever its type, so a write
|
"""The one key-identity judge (variants:ruling:cache-refuses-retype): a key holds one record whatever its type, so a write
|
||||||
|
|
@ -663,18 +718,34 @@ class CultCache:
|
||||||
if held is None:
|
if held is None:
|
||||||
continue
|
continue
|
||||||
type, slots = held
|
type, slots = held
|
||||||
raise SchemaConflictError(
|
raise self._read_only_refusal(self._state.documents[type], key, self._state.envelopes[type][key].schema_id, slots)
|
||||||
self._state.envelopes[type][key].schema_id or type,
|
|
||||||
[self._state.documents[type].catalog_entry().schema_name],
|
@staticmethod
|
||||||
key,
|
def _read_only_refusal(
|
||||||
message=(
|
document: DocumentDefinition[Any], key: str, schema_id: str | None, slots: UnreadSlots
|
||||||
f"{self._state.documents[type].catalog_entry().schema_name} record {key} is read-only: {slots.type_name} was read with "
|
) -> SchemaConflictError:
|
||||||
f"{slots.slot_count} slots where this build declares {slots.declared_count} ({slots.occurrences} object(s) in the "
|
"""The one refusal of a write to a record that carries slots this build would drop: a record the cache holds read-only,
|
||||||
"record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or "
|
or one an update reads from the store."""
|
||||||
"write it from that build."
|
name = document.catalog_entry().schema_name
|
||||||
),
|
return SchemaConflictError(
|
||||||
unread_slots=slots,
|
schema_id or document.type,
|
||||||
)
|
[name],
|
||||||
|
key,
|
||||||
|
message=(
|
||||||
|
f"{name} record {key} is read-only: {slots.type_name} was read with "
|
||||||
|
f"{slots.slot_count} slots where this build declares {slots.declared_count} ({slots.occurrences} object(s) in the "
|
||||||
|
"record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or "
|
||||||
|
"write it from that build."
|
||||||
|
),
|
||||||
|
unread_slots=slots,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _legacy_global_refusal(document: DocumentDefinition[Any], legacy_key: str) -> CultCacheError:
|
||||||
|
return CultCacheError(
|
||||||
|
f"Global {document.type} is held under the legacy key {legacy_key!r}; update cannot copy it. "
|
||||||
|
"put_global writes the global and retires the legacy key."
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _foreign_refusal(record: ForeignRecord) -> SchemaConflictError:
|
def _foreign_refusal(record: ForeignRecord) -> SchemaConflictError:
|
||||||
|
|
@ -707,20 +778,81 @@ class CultCache:
|
||||||
self._assert_global(document)
|
self._assert_global(document)
|
||||||
self._put(document, GLOBAL_KEY, value)
|
self._put(document, GLOBAL_KEY, value)
|
||||||
|
|
||||||
@_mutating
|
def update(
|
||||||
def update(self, document: DocumentDefinition[T], key: str, updater: Any) -> T:
|
self, document: DocumentDefinition[T], key: str, apply: Callable[[T | None], T | None], *, max_attempts: int = 5
|
||||||
return self._update(document, key, updater)
|
) -> UpdateResult[T]:
|
||||||
|
"""Reads the record at the key as its home store holds it, applies `apply` to a fresh copy of it, and commits the result
|
||||||
|
only if the record is still the one that was read; when another writer got there first it reads again and runs `apply`
|
||||||
|
again, up to `max_attempts` times.
|
||||||
|
|
||||||
def _update(self, document: DocumentDefinition[T], key: str, updater: Any) -> T:
|
`apply` may run up to `max_attempts` times, on a fresh copy of the record as the store holds it now (None when the key
|
||||||
current = self.get_required(document, key)
|
holds nothing), which it may change and return. It must not write the same key through this cache or have side effects.
|
||||||
updated = updater(current)
|
Returning None writes nothing (DECLINED). A lost race that outlasts `max_attempts` returns LOST_RACE and does not raise;
|
||||||
self._put(document, key, updated)
|
a refusal, or a raise from `apply`, propagates and is not retried. The write is durable when update returns. The
|
||||||
return updated
|
result's document is the cache's record after the outcome. `put` is the blind write. A cache with no store copies its
|
||||||
|
held instance; a store without `read` and `commit_if` (the JSON lines store among them) refuses before `apply` runs."""
|
||||||
|
if not callable(apply):
|
||||||
|
raise TypeError("CultCache update needs a function to apply.")
|
||||||
|
if not isinstance(max_attempts, int) or isinstance(max_attempts, bool):
|
||||||
|
raise TypeError(f"CultCache update max_attempts must be an int, not {type(max_attempts).__name__}.")
|
||||||
|
if max_attempts < 1:
|
||||||
|
raise ValueError(f"CultCache update max_attempts must be at least 1, not {max_attempts}.")
|
||||||
|
for attempt in range(1, max_attempts + 1):
|
||||||
|
copy, expected = self._read_for_update(document, key)
|
||||||
|
# Outside the cache's lock and the store's: apply may take its time and may write other keys through this cache.
|
||||||
|
next_value = apply(copy)
|
||||||
|
if next_value is None:
|
||||||
|
return UpdateResult(UpdateOutcome.DECLINED, self.get(document, key))
|
||||||
|
if self._commit_update(document, key, next_value, expected) == "committed":
|
||||||
|
return UpdateResult(UpdateOutcome.UPDATED, self.get(document, key))
|
||||||
|
# Display only: the cache shows what the winner left.
|
||||||
|
if self._state.stores_by_type or self._state.generic_store is not None:
|
||||||
|
self.pull_all_backing_stores()
|
||||||
|
return UpdateResult(UpdateOutcome.LOST_RACE, self.get(document, key))
|
||||||
|
|
||||||
@_mutating
|
@_mutating
|
||||||
def update_global(self, document: DocumentDefinition[T], updater: Any) -> T:
|
def _read_for_update(self, document: DocumentDefinition[T], key: str) -> tuple[T | None, _Expected]:
|
||||||
|
"""One attempt's read: the copy to hand `apply` and the condition its commit holds. Every refusal that does not depend on
|
||||||
|
what `apply` returns is raised here, so `apply` never runs for a write that cannot land. Installs nothing."""
|
||||||
|
self._assert_registered(document)
|
||||||
|
self._refuse_at(document, key)
|
||||||
|
legacy_key = self._state.legacy_global_keys.get(document.type)
|
||||||
|
if legacy_key is not None:
|
||||||
|
raise self._legacy_global_refusal(document, legacy_key)
|
||||||
|
store = self._store_for_type(document.type)
|
||||||
|
if store is None:
|
||||||
|
held = self._state.envelopes.get(document.type, {}).get(key)
|
||||||
|
return (None if held is None else document.decode_payload(held.payload)), _Expected(held=held)
|
||||||
|
read, commit_if = getattr(store, "read", None), getattr(store, "commit_if", None)
|
||||||
|
if read is None or commit_if is None:
|
||||||
|
raise CultCacheError(
|
||||||
|
f"Backing store {type(store).__name__} for {document.type} cannot write conditionally; "
|
||||||
|
"update needs a store with read and commit_if."
|
||||||
|
)
|
||||||
|
if document.global_document:
|
||||||
|
for envelope in store.pull_all():
|
||||||
|
if envelope.key != GLOBAL_KEY and self._resolve_document_for_envelope(envelope) is document:
|
||||||
|
raise self._legacy_global_refusal(document, envelope.key)
|
||||||
|
found = read(key)
|
||||||
|
if found is None:
|
||||||
|
return None, _Expected(store=store)
|
||||||
|
envelope, version = found
|
||||||
|
if self._resolve_document_for_envelope(envelope) is not document:
|
||||||
|
raise CultCacheError(f"Record {key} is not a {document.type}; update cannot copy it.")
|
||||||
|
value, unread = self._decode_marked(document, envelope)
|
||||||
|
if unread is not None:
|
||||||
|
raise self._read_only_refusal(document, key, envelope.schema_id, unread)
|
||||||
|
return value, _Expected(store=store, version=version)
|
||||||
|
|
||||||
|
@_mutating
|
||||||
|
def _commit_update(self, document: DocumentDefinition[T], key: str, value: T, expected: _Expected) -> str:
|
||||||
|
return self._put(document, key, value, expected)
|
||||||
|
|
||||||
|
def update_global(
|
||||||
|
self, document: DocumentDefinition[T], apply: Callable[[T | None], T | None], *, max_attempts: int = 5
|
||||||
|
) -> UpdateResult[T]:
|
||||||
self._assert_global(document)
|
self._assert_global(document)
|
||||||
return self._update(document, GLOBAL_KEY, updater)
|
return self.update(document, GLOBAL_KEY, apply, max_attempts=max_attempts)
|
||||||
|
|
||||||
@_mutating
|
@_mutating
|
||||||
def delete(self, document: DocumentDefinition[Any], key: str) -> None:
|
def delete(self, document: DocumentDefinition[Any], key: str) -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,17 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import replace
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable, Iterator, TypeVar
|
from typing import Any, Callable, Iterator, TypeVar
|
||||||
|
|
||||||
|
|
@ -78,6 +82,9 @@ class JsonLinesBackingStore:
|
||||||
def pull_all(self) -> list[CultCacheEnvelope]:
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
||||||
return self._pull() or []
|
return self._pull() or []
|
||||||
|
|
||||||
|
def pull_or_absent(self) -> list[CultCacheEnvelope] | None:
|
||||||
|
return self._pull()
|
||||||
|
|
||||||
def _pull(self) -> list[CultCacheEnvelope] | None:
|
def _pull(self) -> list[CultCacheEnvelope] | None:
|
||||||
"""The envelopes the reader reads, or None when nothing is at the path."""
|
"""The envelopes the reader reads, or None when nothing is at the path."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
|
@ -104,14 +111,23 @@ class JsonLinesBackingStore:
|
||||||
raise ValueError(f"Malformed CultCache JSONL envelope at {self.path}:{line_number}: missing {exc}") from exc
|
raise ValueError(f"Malformed CultCache JSONL envelope at {self.path}:{line_number}: missing {exc}") from exc
|
||||||
return envelopes
|
return envelopes
|
||||||
|
|
||||||
def push(self, envelope: CultCacheEnvelope) -> None:
|
def push(self, envelope: CultCacheEnvelope) -> str | None:
|
||||||
self.push_all([envelope])
|
return self.push_all([envelope])[envelope.key]
|
||||||
|
|
||||||
|
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> dict[str, str]:
|
||||||
|
"""Merges the envelopes, each stored later than any record it replaces, and returns the stamps written by key
|
||||||
|
({} when a soft push skipped)."""
|
||||||
|
stamps: dict[str, str] = {}
|
||||||
|
|
||||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None:
|
|
||||||
def merge(existing: dict[tuple[str, str], CultCacheEnvelope]) -> None:
|
def merge(existing: dict[tuple[str, str], CultCacheEnvelope]) -> None:
|
||||||
for envelope in envelopes:
|
for envelope in envelopes:
|
||||||
|
replaced = existing.get((envelope.type, envelope.key))
|
||||||
|
if replaced is not None:
|
||||||
|
envelope = replace(envelope, stored_at=stored_at_over(envelope.stored_at, replaced.stored_at))
|
||||||
existing[(envelope.type, envelope.key)] = envelope
|
existing[(envelope.type, envelope.key)] = envelope
|
||||||
|
stamps[envelope.key] = envelope.stored_at
|
||||||
self._rewrite(merge, soft)
|
self._rewrite(merge, soft)
|
||||||
|
return stamps
|
||||||
|
|
||||||
def delete(self, type: str, key: str) -> None:
|
def delete(self, type: str, key: str) -> None:
|
||||||
self._rewrite(lambda existing: existing.pop((type, key), None))
|
self._rewrite(lambda existing: existing.pop((type, key), None))
|
||||||
|
|
@ -152,6 +168,9 @@ class SingleFileMessagePackBackingStore:
|
||||||
def pull_all(self) -> list[CultCacheEnvelope]:
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
||||||
return self._pull() or []
|
return self._pull() or []
|
||||||
|
|
||||||
|
def pull_or_absent(self) -> list[CultCacheEnvelope] | None:
|
||||||
|
return self._pull()
|
||||||
|
|
||||||
def _pull(self) -> list[CultCacheEnvelope] | None:
|
def _pull(self) -> list[CultCacheEnvelope] | None:
|
||||||
"""The envelopes the reader reads, or None when nothing is at the path."""
|
"""The envelopes the reader reads, or None when nothing is at the path."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
|
@ -171,23 +190,40 @@ class SingleFileMessagePackBackingStore:
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise StoreUnreadableError(self.path, exc) from exc
|
raise StoreUnreadableError(self.path, exc) from exc
|
||||||
|
|
||||||
def push(self, envelope: CultCacheEnvelope) -> None:
|
def push(self, envelope: CultCacheEnvelope) -> str | None:
|
||||||
with self._write_transaction():
|
with self._write_transaction():
|
||||||
self._replace_all([envelope], self.pull_all())
|
return self._replace_all([envelope], self.pull_all())[envelope.key]
|
||||||
|
|
||||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None:
|
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> dict[str, str]:
|
||||||
"""A soft push writes only where nothing is at the path: a store the reader reads is left byte for byte, and a file it
|
"""A soft push writes only where nothing is at the path: a store the reader reads is left byte for byte, and a file it
|
||||||
cannot read is refused by the same read, as in TS and Rust."""
|
cannot read is refused by the same read, as in TS and Rust. Returns the stamps written by key ({} when skipped)."""
|
||||||
with self._write_transaction():
|
with self._write_transaction():
|
||||||
arrived = self._pull()
|
arrived = self._pull()
|
||||||
if soft and arrived is not None:
|
if soft and arrived is not None:
|
||||||
return
|
return {}
|
||||||
self._replace_all(list(envelopes), arrived or [])
|
return self._replace_all(list(envelopes), arrived or [])
|
||||||
|
|
||||||
def delete(self, type: str, key: str) -> None:
|
def delete(self, type: str, key: str) -> None:
|
||||||
with self._write_transaction():
|
with self._write_transaction():
|
||||||
self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)])
|
self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)])
|
||||||
|
|
||||||
|
def read(self, key: str) -> tuple[CultCacheEnvelope, str] | None:
|
||||||
|
"""The record at the key, of any type, and its content version, from one read; None when the key holds nothing."""
|
||||||
|
for envelope in self._pull() or []:
|
||||||
|
if envelope.key == key:
|
||||||
|
return envelope, record_version(envelope)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def commit_if(self, envelope: CultCacheEnvelope, expected: str | None) -> str | None:
|
||||||
|
"""Writes the envelope only if the record at its key (any type) is still the one whose version is `expected`
|
||||||
|
(None: no record), judged under the write lock; returns the stored_at written, or None with nothing written."""
|
||||||
|
with self._write_transaction():
|
||||||
|
arrived = self._pull() or []
|
||||||
|
current = next((item for item in arrived if item.key == envelope.key), None)
|
||||||
|
if (None if current is None else record_version(current)) != expected:
|
||||||
|
return None
|
||||||
|
return self._replace_all([envelope], arrived)[envelope.key]
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _write_transaction(self) -> Iterator[None]:
|
def _write_transaction(self) -> Iterator[None]:
|
||||||
"""The thread lock, then the cross-process lock on `<path>.lock`, held across the read and the replace so two
|
"""The thread lock, then the cross-process lock on `<path>.lock`, held across the read and the replace so two
|
||||||
|
|
@ -197,8 +233,10 @@ class SingleFileMessagePackBackingStore:
|
||||||
with store_write_lock(self.path, self.lock_timeout):
|
with store_write_lock(self.path, self.lock_timeout):
|
||||||
yield
|
yield
|
||||||
|
|
||||||
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None:
|
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> dict[str, str]:
|
||||||
commit_store_file(self.path, msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True))
|
snapshot, stamps = _encode_snapshot(supplied, arrived, self._format)
|
||||||
|
commit_store_file(self.path, msgpack.packb(snapshot, use_bin_type=True))
|
||||||
|
return stamps
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -299,7 +337,7 @@ def written_owns(held: CultCacheEnvelope, written: CultCacheEnvelope) -> bool:
|
||||||
|
|
||||||
def _encode_snapshot(
|
def _encode_snapshot(
|
||||||
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
|
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
|
||||||
) -> list[Any]:
|
) -> tuple[list[Any], dict[str, str]]:
|
||||||
# A key names one record whatever its type, so a supplied record replaces the arrived one by key alone, and only of the
|
# A key names one record whatever its type, so a supplied record replaces the arrived one by key alone, and only of the
|
||||||
# type that owns it: retyping a key is a removal and an add, so a record another type owns under the key refuses the write.
|
# type that owns it: retyping a key is a removal and an add, so a record another type owns under the key refuses the write.
|
||||||
held_by_key = {envelope.key: envelope for envelope in arrived}
|
held_by_key = {envelope.key: envelope for envelope in arrived}
|
||||||
|
|
@ -308,8 +346,12 @@ def _encode_snapshot(
|
||||||
for held in (landing.get(envelope.key), held_by_key.get(envelope.key)):
|
for held in (landing.get(envelope.key), held_by_key.get(envelope.key)):
|
||||||
if held is not None and not written_owns(held, envelope):
|
if held is not None and not written_owns(held, envelope):
|
||||||
raise retyped(held, envelope)
|
raise retyped(held, envelope)
|
||||||
|
held = held_by_key.get(envelope.key)
|
||||||
|
if held is not None:
|
||||||
|
envelope = replace(envelope, stored_at=stored_at_over(envelope.stored_at, held.stored_at))
|
||||||
landing[envelope.key] = envelope
|
landing[envelope.key] = envelope
|
||||||
supplied = list(landing.values())
|
supplied = list(landing.values())
|
||||||
|
stamps = {envelope.key: envelope.stored_at for envelope in supplied}
|
||||||
replaced = set(landing)
|
replaced = set(landing)
|
||||||
kept = [envelope for envelope in arrived if envelope.key not in replaced]
|
kept = [envelope for envelope in arrived if envelope.key not in replaced]
|
||||||
envelopes = [*kept, *supplied]
|
envelopes = [*kept, *supplied]
|
||||||
|
|
@ -324,7 +366,7 @@ def _encode_snapshot(
|
||||||
[envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
|
[envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
|
||||||
for envelope in sorted(envelopes, key=lambda item: item.key)
|
for envelope in sorted(envelopes, key=lambda item: item.key)
|
||||||
]
|
]
|
||||||
return [format_version, catalog, records]
|
return [format_version, catalog, records], stamps
|
||||||
|
|
||||||
def commit_store_file(path: Path, data: bytes) -> None:
|
def commit_store_file(path: Path, data: bytes) -> None:
|
||||||
"""Writes `data` to a unique staged file beside the store, fsyncs it, replaces the store with it and, off Windows,
|
"""Writes `data` to a unique staged file beside the store, fsyncs it, replaces the store with it and, off Windows,
|
||||||
|
|
@ -495,6 +537,8 @@ def store_write_lock(path: Path, timeout: float = LOCK_TIMEOUT_SECONDS) -> Itera
|
||||||
raise TimeoutError(f"CultCache timed out after {timeout} s waiting for the store lock {lock_path}.")
|
raise TimeoutError(f"CultCache timed out after {timeout} s waiting for the store lock {lock_path}.")
|
||||||
time.sleep(_LOCK_POLL_SECONDS)
|
time.sleep(_LOCK_POLL_SECONDS)
|
||||||
try:
|
try:
|
||||||
|
# A link may have been put at the path while the lock was awaited; the look above only saw the path before.
|
||||||
|
_refuse_linked(path)
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
release()
|
release()
|
||||||
|
|
@ -731,6 +775,65 @@ def _schema_id_for(envelope: CultCacheEnvelope) -> str:
|
||||||
return envelope.schema_id or envelope.type
|
return envelope.schema_id or envelope.type
|
||||||
|
|
||||||
|
|
||||||
|
_STORED_AT_SHAPE = re.compile(
|
||||||
|
r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]{1,9}))?(?:(Z)|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_stored_at(text: Any) -> datetime | None:
|
||||||
|
"""The UTC instant a stored_at text names, or None. Only the shape the runtimes write is a stamp: an RFC 3339 date-time with T,
|
||||||
|
an optional fraction of 1 to 9 digits (cut to the microsecond) and Z or a numeric offset of hours 00-23 and minutes 00-59. Checked here, not left to
|
||||||
|
fromisoformat, whose accepted set differs by interpreter (3.10 and 3.13) and from Rust and TypeScript. An instant outside
|
||||||
|
datetime's range is not a stamp."""
|
||||||
|
if not isinstance(text, str):
|
||||||
|
return None
|
||||||
|
shape = _STORED_AT_SHAPE.fullmatch(text)
|
||||||
|
if shape is None:
|
||||||
|
return None
|
||||||
|
year, month, day, hour, minute, second, fraction, zulu, sign, offset_hours, offset_minutes = shape.groups()
|
||||||
|
try:
|
||||||
|
offset = timezone.utc if zulu else timezone(
|
||||||
|
(-1 if sign == "-" else 1) * timedelta(hours=int(offset_hours), minutes=int(offset_minutes))
|
||||||
|
)
|
||||||
|
local = datetime(
|
||||||
|
int(year), int(month), int(day), int(hour), int(minute), int(second), int(((fraction or "") + "000000")[:6]), offset
|
||||||
|
)
|
||||||
|
return local.astimezone(timezone.utc)
|
||||||
|
except (ValueError, OverflowError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def mint_stored_at(previous: str | None, now: datetime) -> str:
|
||||||
|
"""A stored_at later than `previous` (C# MintStoredAt): `now`, or the microsecond after `previous` when `now` is not later.
|
||||||
|
A previous that does not parse leaves `now`. Raises ValueError, nothing minted, when `previous` is the last instant a stamp
|
||||||
|
can name, so a write over it is refused and writes nothing."""
|
||||||
|
now = now.astimezone(timezone.utc)
|
||||||
|
last = parse_stored_at(previous)
|
||||||
|
if last is not None and now <= last:
|
||||||
|
try:
|
||||||
|
now = last + timedelta(microseconds=1)
|
||||||
|
except OverflowError:
|
||||||
|
raise ValueError(f"stored_at {previous!r} is the last instant a stamp can name; a write over it cannot be stored later") from None
|
||||||
|
return now.isoformat(timespec="microseconds")
|
||||||
|
|
||||||
|
|
||||||
|
def stored_at_over(staged: str, replaced: str) -> str:
|
||||||
|
"""The stored_at a record replacing `replaced` is written with (C# StoredAtOver): `staged` when both parse and it is strictly
|
||||||
|
later, else one minted over `replaced`. No content is compared: every replacing write is stamped later."""
|
||||||
|
later, earlier = parse_stored_at(staged), parse_stored_at(replaced)
|
||||||
|
if later is not None and earlier is not None and later > earlier:
|
||||||
|
return staged
|
||||||
|
return mint_stored_at(replaced, datetime.now(timezone.utc))
|
||||||
|
|
||||||
|
|
||||||
|
def record_version(envelope: CultCacheEnvelope) -> str:
|
||||||
|
"""The content version of a stored record: the lowercase hex SHA-256 of its persisted encoding, the array
|
||||||
|
[key, schema id, stored_at, payload] exactly as the single-file store writes it
|
||||||
|
(variants:ruling:write-set-content-version-declared-ownership)."""
|
||||||
|
record = [envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
|
||||||
|
return hashlib.sha256(msgpack.packb(record, use_bin_type=True)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _default_catalog_entry(envelope: CultCacheEnvelope) -> CultCacheSchemaCatalogEntry:
|
def _default_catalog_entry(envelope: CultCacheEnvelope) -> CultCacheSchemaCatalogEntry:
|
||||||
schema_id = _schema_id_for(envelope)
|
schema_id = _schema_id_for(envelope)
|
||||||
return CultCacheSchemaCatalogEntry(
|
return CultCacheSchemaCatalogEntry(
|
||||||
|
|
|
||||||
38
packages/cultcache-py/tests/test_content_version_vector.py
Normal file
38
packages/cultcache-py/tests/test_content_version_vector.py
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
|
||||||
|
from cultcache_py import SingleFileMessagePackBackingStore
|
||||||
|
from cultcache_py.stores import record_version
|
||||||
|
|
||||||
|
# variants:ruling:write-set-content-version-declared-ownership. The vector in tests/vectors/content-version is written only by
|
||||||
|
# the C# reference; every row of it is computed here from the store file, by the store's read and by record_version.
|
||||||
|
VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "content-version"
|
||||||
|
|
||||||
|
|
||||||
|
class ContentVersionVectorTests(unittest.TestCase):
|
||||||
|
def test_every_row_of_the_vector_equals_this_runtimes_version(self) -> None:
|
||||||
|
rows = msgpack.unpackb((VECTORS / "versions.msgpack").read_bytes(), raw=False)
|
||||||
|
self.assertEqual([key for key, _ in rows], ["alpha", "beta"])
|
||||||
|
directory = Path(tempfile.mkdtemp())
|
||||||
|
self.addCleanup(shutil.rmtree, directory, ignore_errors=True)
|
||||||
|
copy = directory / "store.cc"
|
||||||
|
shutil.copyfile(VECTORS / "store.cc", copy)
|
||||||
|
store = SingleFileMessagePackBackingStore(copy)
|
||||||
|
by_key = {envelope.key: envelope for envelope in store.pull_all()}
|
||||||
|
for key, version in rows:
|
||||||
|
with self.subTest(key=key):
|
||||||
|
found = store.read(key)
|
||||||
|
self.assertIsNotNone(found)
|
||||||
|
self.assertEqual(found[1], version)
|
||||||
|
self.assertEqual(record_version(by_key[key]), version)
|
||||||
|
self.assertIsNone(store.read("gamma"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -86,7 +86,7 @@ class CultCacheTests(unittest.TestCase):
|
||||||
)
|
)
|
||||||
cache.pull_all_backing_stores()
|
cache.pull_all_backing_stores()
|
||||||
cache.put(document, "item:potion", Item(name="Potion", category="Consumable", value=50))
|
cache.put(document, "item:potion", Item(name="Potion", category="Consumable", value=50))
|
||||||
cache.update(document, "item:potion", lambda item: Item(item.name, item.category, item.value + 5))
|
cache.put(document, "item:potion", Item(name="Potion", category="Consumable", value=55))
|
||||||
self.assertEqual(cache.get_required(document, "item:potion").value, 55)
|
self.assertEqual(cache.get_required(document, "item:potion").value, 55)
|
||||||
cache.delete(document, "item:potion")
|
cache.delete(document, "item:potion")
|
||||||
self.assertIsNone(cache.get(document, "item:potion"))
|
self.assertIsNone(cache.get(document, "item:potion"))
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from cultcache_py import UpdateOutcome
|
||||||
|
|
||||||
# snippet:begin
|
# snippet:begin
|
||||||
import tempfile
|
import tempfile
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
|
|
@ -51,15 +53,44 @@ def first_call() -> Item:
|
||||||
# snippet:end
|
# snippet:end
|
||||||
|
|
||||||
|
|
||||||
|
# snippet:update:begin
|
||||||
|
def restock(path: Path):
|
||||||
|
cache = (
|
||||||
|
CultCache.builder()
|
||||||
|
.register_document_type(item_doc)
|
||||||
|
.add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
return cache.update(item_doc, "item:potion", lambda item: Item("Potion", "Consumable", (item.value if item else 0) + 10))
|
||||||
|
# snippet:update:end
|
||||||
|
|
||||||
|
|
||||||
class FirstCallTests(unittest.TestCase):
|
class FirstCallTests(unittest.TestCase):
|
||||||
def test_first_call_round_trips_through_a_second_cache(self) -> None:
|
def test_first_call_round_trips_through_a_second_cache(self) -> None:
|
||||||
self.assertEqual(first_call(), Item("Potion", "Consumable", 50))
|
self.assertEqual(first_call(), Item("Potion", "Consumable", 50))
|
||||||
|
|
||||||
def test_docs_show_the_tested_snippet(self) -> None:
|
def test_update_with_retry_through_a_second_cache(self) -> None:
|
||||||
|
path = Path(tempfile.mkdtemp()) / "restock.cc"
|
||||||
|
restock(path)
|
||||||
|
result = restock(path)
|
||||||
|
self.assertIs(result.outcome, UpdateOutcome.UPDATED)
|
||||||
|
reopened = (
|
||||||
|
CultCache.builder()
|
||||||
|
.register_document_type(item_doc)
|
||||||
|
.add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||||
|
.build()
|
||||||
|
)
|
||||||
|
reopened.pull_all_backing_stores()
|
||||||
|
self.assertEqual(reopened.get_required(item_doc, "item:potion"), Item("Potion", "Consumable", 20))
|
||||||
|
|
||||||
|
def test_docs_show_the_tested_snippets(self) -> None:
|
||||||
source = Path(__file__).read_text(encoding="utf-8")
|
source = Path(__file__).read_text(encoding="utf-8")
|
||||||
snippet = source.split("# snippet:begin\n", 1)[1].split("# snippet:end\n", 1)[0]
|
|
||||||
docs = (Path(__file__).parents[3] / "docs" / "start" / "README.md").read_text(encoding="utf-8")
|
docs = (Path(__file__).parents[3] / "docs" / "start" / "README.md").read_text(encoding="utf-8")
|
||||||
self.assertIn("```python\n" + snippet + "```", docs)
|
for region in ("snippet", "snippet:update"):
|
||||||
|
with self.subTest(region=region):
|
||||||
|
snippet = source.split(f"# {region}:begin\n", 1)[1].split(f"# {region}:end\n", 1)[0]
|
||||||
|
self.assertIn("```python\n" + snippet + "```", docs)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -69,8 +69,33 @@ class SoftPushAllTests(unittest.TestCase):
|
||||||
JsonLinesBackingStore(self.path).push_all([envelope("a")], soft=True)
|
JsonLinesBackingStore(self.path).push_all([envelope("a")], soft=True)
|
||||||
self.assertEqual(junk, self.path.read_bytes())
|
self.assertEqual(junk, self.path.read_bytes())
|
||||||
|
|
||||||
def test_the_protocol_default_push_all_skips_what_its_pull_read_empty_included(self) -> None:
|
def test_the_protocol_default_push_all_decides_soft_by_what_pull_or_absent_says(self) -> None:
|
||||||
class Memory(BackingStore):
|
class Memory(BackingStore):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.items: list[CultCacheEnvelope] | None = None # None: nothing at the path
|
||||||
|
|
||||||
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
||||||
|
return list(self.items or [])
|
||||||
|
|
||||||
|
def pull_or_absent(self) -> list[CultCacheEnvelope] | None:
|
||||||
|
return None if self.items is None else list(self.items)
|
||||||
|
|
||||||
|
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
||||||
|
self.items = list(envelopes)
|
||||||
|
|
||||||
|
store = Memory()
|
||||||
|
store.push_all([envelope("a")], soft=True) # nothing at the path: writes
|
||||||
|
self.assertEqual(["a"], [item.key for item in store.items or []])
|
||||||
|
store.push_all([envelope("b")], soft=True) # a store with records: skips
|
||||||
|
self.assertEqual(["a"], [item.key for item in store.items or []])
|
||||||
|
store.items = []
|
||||||
|
store.push_all([envelope("b")], soft=True) # a store the reader read, empty: skips
|
||||||
|
self.assertEqual([], store.items)
|
||||||
|
store.push_all([envelope("b")]) # plain: lands
|
||||||
|
self.assertEqual(["b"], [item.key for item in store.items])
|
||||||
|
|
||||||
|
def test_a_store_that_only_pulls_all_is_read_so_a_soft_push_skips_it(self) -> None:
|
||||||
|
class PullOnly(BackingStore):
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.items: list[CultCacheEnvelope] = []
|
self.items: list[CultCacheEnvelope] = []
|
||||||
|
|
||||||
|
|
@ -80,14 +105,18 @@ class SoftPushAllTests(unittest.TestCase):
|
||||||
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
||||||
self.items = list(envelopes)
|
self.items = list(envelopes)
|
||||||
|
|
||||||
store = Memory()
|
store = PullOnly()
|
||||||
store.push_all([envelope("a")], soft=True) # the pull read an empty store: skips
|
store.push_all([envelope("a")], soft=True)
|
||||||
self.assertEqual([], store.items)
|
self.assertEqual([], store.items)
|
||||||
store.push_all([envelope("a")]) # plain: lands
|
|
||||||
store.push_all([envelope("b")], soft=True) # a store with records: skips
|
def test_the_single_file_and_jsonl_stores_say_absent_and_read_through_pull_or_absent(self) -> None:
|
||||||
self.assertEqual(["a"], [item.key for item in store.items])
|
for make in (SingleFileMessagePackBackingStore, JsonLinesBackingStore):
|
||||||
store.push_all([envelope("b")])
|
with self.subTest(make.__name__):
|
||||||
self.assertEqual(["a", "b"], sorted(item.key for item in store.items))
|
path = self.dir / make.__name__
|
||||||
|
store = make(path)
|
||||||
|
self.assertIsNone(store.pull_or_absent())
|
||||||
|
store.push_all([envelope("a")])
|
||||||
|
self.assertEqual(["a"], [item.key for item in store.pull_or_absent() or []])
|
||||||
|
|
||||||
def test_the_protocol_default_push_all_refuses_where_its_pull_refuses(self) -> None:
|
def test_the_protocol_default_push_all_refuses_where_its_pull_refuses(self) -> None:
|
||||||
class Unreadable(BackingStore):
|
class Unreadable(BackingStore):
|
||||||
|
|
@ -96,6 +125,9 @@ class SoftPushAllTests(unittest.TestCase):
|
||||||
def pull_all(self) -> list[CultCacheEnvelope]:
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
||||||
raise StoreUnreadableError(Path("x"), ValueError("junk"))
|
raise StoreUnreadableError(Path("x"), ValueError("junk"))
|
||||||
|
|
||||||
|
def pull_or_absent(self) -> list[CultCacheEnvelope] | None:
|
||||||
|
raise StoreUnreadableError(Path("x"), ValueError("junk"))
|
||||||
|
|
||||||
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
||||||
Unreadable.replaced = True
|
Unreadable.replaced = True
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,25 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import select
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
from cultcache_py import SingleFileMessagePackBackingStore, stores
|
from cultcache_py import SingleFileMessagePackBackingStore, stores
|
||||||
|
from cultcache_py.stores import StoreLinkedError, store_write_lock
|
||||||
from cultcache_py.backing_store import CultCacheEnvelope
|
from cultcache_py.backing_store import CultCacheEnvelope
|
||||||
|
|
||||||
REPO = Path(__file__).resolve().parents[3]
|
REPO = Path(__file__).resolve().parents[3]
|
||||||
TS_STORE_FILE = REPO / "packages" / "cultcache-ts" / "dist" / "store-file.js"
|
TS_STORE_FILE = REPO / "packages" / "cultcache-ts" / "dist" / "store-file.js"
|
||||||
HOLD_SECONDS = 1.5
|
HOLD_SECONDS = 1.5
|
||||||
|
HELD_WAIT_SECONDS = 3
|
||||||
|
|
||||||
# A holder that takes the lock the way the Rust runtime does (an exclusive file lock), not through this package.
|
# A holder that takes the lock the way the Rust runtime does (an exclusive file lock), not through this package.
|
||||||
PY_HOLDER = """
|
PY_HOLDER = """
|
||||||
|
|
@ -41,7 +45,14 @@ def envelope(key: str) -> CultCacheEnvelope:
|
||||||
|
|
||||||
def start_holder(argv: list[str]) -> subprocess.Popen[str]:
|
def start_holder(argv: list[str]) -> subprocess.Popen[str]:
|
||||||
holder = subprocess.Popen(argv, stdout=subprocess.PIPE, text=True)
|
holder = subprocess.Popen(argv, stdout=subprocess.PIPE, text=True)
|
||||||
assert holder.stdout is not None and holder.stdout.readline().strip() == "HELD"
|
assert holder.stdout is not None
|
||||||
|
ready, _, _ = select.select([holder.stdout], [], [], HELD_WAIT_SECONDS) # a holder blocked on a leaked lock never prints
|
||||||
|
if not ready:
|
||||||
|
holder.kill()
|
||||||
|
holder.wait()
|
||||||
|
holder.stdout.close()
|
||||||
|
raise AssertionError(f"the holder never took {argv[-2]}.lock within {HELD_WAIT_SECONDS} s: an earlier writer leaked the lock")
|
||||||
|
assert holder.stdout.readline().strip() == "HELD"
|
||||||
holder.held_at = time.monotonic() # type: ignore[attr-defined]
|
holder.held_at = time.monotonic() # type: ignore[attr-defined]
|
||||||
return holder
|
return holder
|
||||||
|
|
||||||
|
|
@ -73,6 +84,48 @@ class StoreLockTests(unittest.TestCase):
|
||||||
holder = start_holder(["node", "-e", TS_HOLDER, str(TS_STORE_FILE), str(self.path), str(HOLD_SECONDS)])
|
holder = start_holder(["node", "-e", TS_HOLDER, str(TS_STORE_FILE), str(self.path), str(HOLD_SECONDS)])
|
||||||
self.assert_write_waits_for(holder)
|
self.assert_write_waits_for(holder)
|
||||||
|
|
||||||
|
def test_a_link_put_while_a_writer_waits_on_the_lock_is_refused_once_it_holds_it(self) -> None:
|
||||||
|
target = self.dir / "volume" / "store.cc"
|
||||||
|
target.parent.mkdir()
|
||||||
|
SingleFileMessagePackBackingStore(target).push(envelope("a"))
|
||||||
|
before = target.read_bytes()
|
||||||
|
writes = {
|
||||||
|
"push": lambda store: store.push(envelope("b")),
|
||||||
|
"push_all": lambda store: store.push_all([envelope("b")]),
|
||||||
|
"delete": lambda store: store.delete("t", "a"),
|
||||||
|
}
|
||||||
|
missing = self.dir / "volume" / "missing.cc"
|
||||||
|
for (name, write), link_to in [(item, to) for item in writes.items() for to in (target, missing)]:
|
||||||
|
with self.subTest(name, dangling=link_to == missing):
|
||||||
|
if os.path.lexists(self.path):
|
||||||
|
os.unlink(self.path)
|
||||||
|
holder = start_holder([sys.executable, "-c", PY_HOLDER, str(self.path), str(HOLD_SECONDS)])
|
||||||
|
self.addCleanup(holder.wait)
|
||||||
|
self.addCleanup(holder.stdout.close) # type: ignore[union-attr]
|
||||||
|
outcome: list[BaseException | None] = []
|
||||||
|
|
||||||
|
def run() -> None:
|
||||||
|
try:
|
||||||
|
write(SingleFileMessagePackBackingStore(self.path, lock_timeout=10))
|
||||||
|
outcome.append(None)
|
||||||
|
except BaseException as exc: # noqa: BLE001 - the thread hands the outcome to the test
|
||||||
|
outcome.append(exc)
|
||||||
|
|
||||||
|
writer = threading.Thread(target=run)
|
||||||
|
writer.start()
|
||||||
|
time.sleep(0.3)
|
||||||
|
os.symlink(link_to, self.path) # after the writer's pre-lock look, while it waits
|
||||||
|
writer.join()
|
||||||
|
holder.wait()
|
||||||
|
self.assertEqual(1, len(outcome))
|
||||||
|
self.assertIsInstance(outcome[0], StoreLinkedError)
|
||||||
|
self.assertTrue(os.path.islink(self.path), "the link was replaced")
|
||||||
|
self.assertEqual(before, target.read_bytes())
|
||||||
|
self.assertFalse(missing.exists(), "the dangling target was created")
|
||||||
|
os.unlink(self.path) # a link is refused before the lock, so remove it to probe the writer's own lock file
|
||||||
|
with store_write_lock(self.path, timeout=0.2): # the lock is free
|
||||||
|
pass
|
||||||
|
|
||||||
def test_the_lock_is_released_after_each_write(self) -> None:
|
def test_the_lock_is_released_after_each_write(self) -> None:
|
||||||
store = SingleFileMessagePackBackingStore(self.path, lock_timeout=0.2)
|
store = SingleFileMessagePackBackingStore(self.path, lock_timeout=0.2)
|
||||||
for key in "abc":
|
for key in "abc":
|
||||||
|
|
|
||||||
|
|
@ -148,7 +148,6 @@ class UniqueIndexTests(unittest.TestCase):
|
||||||
"put": lambda cache, d, k, i: cache.put(d, k, i),
|
"put": lambda cache, d, k, i: cache.put(d, k, i),
|
||||||
"put_envelope": lambda cache, d, k, i: cache.put_envelope(d, envelope(d, k, i)),
|
"put_envelope": lambda cache, d, k, i: cache.put_envelope(d, envelope(d, k, i)),
|
||||||
"put_envelopes": lambda cache, d, k, i: cache.put_envelopes(d, [envelope(d, k, i)]),
|
"put_envelopes": lambda cache, d, k, i: cache.put_envelopes(d, [envelope(d, k, i)]),
|
||||||
"update": lambda cache, d, k, i: cache.update(d, k, lambda _held: i),
|
|
||||||
}
|
}
|
||||||
for name, write in paths.items():
|
for name, write in paths.items():
|
||||||
with self.subTest(alias=alias, path=name):
|
with self.subTest(alias=alias, path=name):
|
||||||
|
|
|
||||||
731
packages/cultcache-py/tests/test_update.py
Normal file
731
packages/cultcache-py/tests/test_update.py
Normal file
|
|
@ -0,0 +1,731 @@
|
||||||
|
"""CultCache.update (variants:cut_spec:cut-update-with-retry-py): read the record as its home store holds it, apply a function,
|
||||||
|
commit only if the record is unchanged, run the function again on a lost race. Mirrors C# UpdateTests and the TS update tests."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from cultcache_py import (
|
||||||
|
CultCache,
|
||||||
|
CultCacheEnvelope,
|
||||||
|
JsonLinesBackingStore,
|
||||||
|
SchemaConflictError,
|
||||||
|
SingleFileMessagePackBackingStore,
|
||||||
|
UpdateOutcome,
|
||||||
|
UpdateResult,
|
||||||
|
database_entry_field,
|
||||||
|
define_database_entry_type,
|
||||||
|
define_document_type,
|
||||||
|
leaves as t,
|
||||||
|
)
|
||||||
|
from cultcache_py.cache import GLOBAL_KEY, CultCacheError
|
||||||
|
from cultcache_py.stores import parse_stored_at, mint_stored_at, record_version, stored_at_over
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Item:
|
||||||
|
name: str = "n"
|
||||||
|
value: int = 0
|
||||||
|
sku: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _doc(type: str, **extra):
|
||||||
|
return define_document_type(type, encode=asdict, decode=lambda raw: Item(**raw), **extra)
|
||||||
|
|
||||||
|
|
||||||
|
item_doc = _doc("item")
|
||||||
|
other_doc = _doc("other")
|
||||||
|
sku_doc = _doc("skuitem", indexes={"sku": "sku"})
|
||||||
|
global_doc = _doc("gitem", global_document=True)
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Wide:
|
||||||
|
name: str = ""
|
||||||
|
extra: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Narrow:
|
||||||
|
name: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
ALPHA = "2099-01-01T00:00:00.0000000Z"
|
||||||
|
|
||||||
|
|
||||||
|
class Fixture(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.dir = Path(tempfile.mkdtemp())
|
||||||
|
self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True)
|
||||||
|
self.path = self.dir / "state.cc"
|
||||||
|
|
||||||
|
def cache(self, *documents, path: Path | None = None, store=True) -> CultCache:
|
||||||
|
builder = CultCache.builder()
|
||||||
|
for document in documents or (item_doc,):
|
||||||
|
builder.register_document_type(document)
|
||||||
|
if store:
|
||||||
|
builder.add_generic_store(SingleFileMessagePackBackingStore(path or self.path))
|
||||||
|
cache = builder.build()
|
||||||
|
if store:
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
return cache
|
||||||
|
|
||||||
|
def stored(self, key: str = "k", document=item_doc):
|
||||||
|
"""What a fresh cache reads at the key."""
|
||||||
|
return self.cache(document).get(document, key)
|
||||||
|
|
||||||
|
def bytes(self) -> bytes | None:
|
||||||
|
return self.path.read_bytes() if self.path.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
class Counting:
|
||||||
|
"""An apply that records what it was given."""
|
||||||
|
|
||||||
|
def __init__(self, fn=None) -> None:
|
||||||
|
self.seen: list = []
|
||||||
|
self.fn = fn or (lambda item, calls: Item("n", (item.value if item else 0) + 1))
|
||||||
|
|
||||||
|
def __call__(self, item):
|
||||||
|
self.seen.append(None if item is None else item.value)
|
||||||
|
# An always-losing apply must fail by assertion when the attempt bound is gone, not hang.
|
||||||
|
assert len(self.seen) <= 10, "apply ran past the attempt bound"
|
||||||
|
return self.fn(item, len(self.seen))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def calls(self) -> int:
|
||||||
|
return len(self.seen)
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateTests(Fixture):
|
||||||
|
def test_two_writers_updating_lose_no_increment(self) -> None:
|
||||||
|
results: list[UpdateResult] = []
|
||||||
|
calls = [0]
|
||||||
|
lock = threading.Lock()
|
||||||
|
a, b = self.cache(), self.cache()
|
||||||
|
a.put(item_doc, "k", Item("n", 0))
|
||||||
|
|
||||||
|
def bump(item):
|
||||||
|
with lock:
|
||||||
|
calls[0] += 1
|
||||||
|
time.sleep(0)
|
||||||
|
return Item("n", item.value + 1)
|
||||||
|
|
||||||
|
def run(cache) -> None:
|
||||||
|
for _ in range(50):
|
||||||
|
results.append(cache.update(item_doc, "k", bump, max_attempts=200))
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=run, args=(cache,)) for cache in (a, b)]
|
||||||
|
for thread in threads:
|
||||||
|
thread.start()
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
self.assertTrue(all(result.outcome is UpdateOutcome.UPDATED for result in results))
|
||||||
|
self.assertEqual(self.stored().value, 100)
|
||||||
|
self.assertGreater(calls[0], 100)
|
||||||
|
|
||||||
|
def test_a_lost_attempt_reruns_apply_on_the_fresh_value(self) -> None:
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
apply = Counting(lambda item, calls: (rival.put(item_doc, "k", Item("n", 100)) if calls == 1 else None) or Item("n", item.value + 1))
|
||||||
|
result = cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.seen, [1, 100])
|
||||||
|
self.assertIs(result.outcome, UpdateOutcome.UPDATED)
|
||||||
|
self.assertEqual(result.document.value, 101)
|
||||||
|
self.assertEqual(self.stored().value, 101)
|
||||||
|
|
||||||
|
def test_exhausted_attempts_return_lost_race(self) -> None:
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
apply = Counting(lambda item, calls: rival.put(item_doc, "k", Item("n", 100 * calls)) or Item("n", -1))
|
||||||
|
result = cache.update(item_doc, "k", apply, max_attempts=3)
|
||||||
|
self.assertEqual(apply.calls, 3)
|
||||||
|
self.assertIs(result.outcome, UpdateOutcome.LOST_RACE)
|
||||||
|
self.assertEqual(result.document.value, 300) # the pull before LOST_RACE shows the winner
|
||||||
|
self.assertEqual(self.stored().value, 300)
|
||||||
|
|
||||||
|
def test_max_attempts_bounds(self) -> None:
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
|
||||||
|
def always_losing(item, calls):
|
||||||
|
rival.put(item_doc, "k", Item("n", 100 + calls))
|
||||||
|
return Item("n", -1)
|
||||||
|
|
||||||
|
apply = Counting(always_losing)
|
||||||
|
result = cache.update(item_doc, "k", apply, max_attempts=1)
|
||||||
|
self.assertEqual((apply.calls, result.outcome), (1, UpdateOutcome.LOST_RACE))
|
||||||
|
for bad, error in ((0, ValueError), (-1, ValueError), (True, TypeError), (1.5, TypeError), (None, TypeError)):
|
||||||
|
with self.subTest(max_attempts=bad):
|
||||||
|
untouched = Counting()
|
||||||
|
with self.assertRaises(error):
|
||||||
|
cache.update(item_doc, "k", untouched, max_attempts=bad)
|
||||||
|
self.assertEqual(untouched.calls, 0)
|
||||||
|
with self.assertRaises(TypeError):
|
||||||
|
cache.update(item_doc, "k", "not callable")
|
||||||
|
with self.assertRaises(TypeError):
|
||||||
|
cache.update(item_doc, "k", None)
|
||||||
|
|
||||||
|
def test_apply_gets_a_fresh_copy(self) -> None:
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
arguments: list[Item] = []
|
||||||
|
|
||||||
|
def apply(item):
|
||||||
|
arguments.append(item)
|
||||||
|
self.assertIsNot(item, cache.get(item_doc, "k"))
|
||||||
|
if len(arguments) == 1:
|
||||||
|
item.value = 777 # changing the copy in place changes nothing the cache or the next attempt sees
|
||||||
|
rival.put(item_doc, "k", Item("n", 5))
|
||||||
|
self.assertEqual(cache.get(item_doc, "k").value, 1)
|
||||||
|
return Item("n", item.value + 1)
|
||||||
|
|
||||||
|
result = cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual([item.value for item in arguments], [777, 5])
|
||||||
|
self.assertIsNot(arguments[0], arguments[1])
|
||||||
|
self.assertEqual(result.document.value, 6)
|
||||||
|
|
||||||
|
def test_an_in_place_change_to_the_held_value_is_neither_applied_nor_persisted(self) -> None:
|
||||||
|
cache = self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
cache.get(item_doc, "k").value = 999
|
||||||
|
apply = Counting(lambda item, calls: Item("n", item.value + 1))
|
||||||
|
result = cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.seen, [1])
|
||||||
|
self.assertEqual(self.stored().value, 2)
|
||||||
|
result.document.value = 999
|
||||||
|
cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.seen, [1, 2])
|
||||||
|
self.assertEqual(self.stored().value, 3)
|
||||||
|
|
||||||
|
bare = self.cache(store=False) # a cache with no store copies its held instance
|
||||||
|
bare.put(item_doc, "k", Item("n", 1))
|
||||||
|
bare.get(item_doc, "k").value = 999
|
||||||
|
seen = Counting(lambda item, calls: Item("n", item.value + 1))
|
||||||
|
self.assertEqual(bare.update(item_doc, "k", seen).document.value, 2)
|
||||||
|
self.assertEqual(seen.seen, [1])
|
||||||
|
|
||||||
|
def test_none_declines(self) -> None:
|
||||||
|
cache = self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
before = self.bytes()
|
||||||
|
result = cache.update(item_doc, "k", lambda item: None)
|
||||||
|
self.assertIs(result.outcome, UpdateOutcome.DECLINED)
|
||||||
|
self.assertEqual(result.document, cache.get(item_doc, "k"))
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
absent = cache.update(item_doc, "missing", lambda item: None)
|
||||||
|
self.assertEqual((absent.outcome, absent.document), (UpdateOutcome.DECLINED, None))
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
empty = Fixture.cache(self, path=self.dir / "empty.cc")
|
||||||
|
self.assertEqual(empty.update(item_doc, "k", lambda item: None).outcome, UpdateOutcome.DECLINED)
|
||||||
|
self.assertFalse((self.dir / "empty.cc").exists())
|
||||||
|
|
||||||
|
def test_a_declined_or_lost_attempt_installs_nothing(self) -> None:
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
rival.put(item_doc, "k", Item("n", 5))
|
||||||
|
self.assertIs(cache.update(item_doc, "k", lambda item: None).outcome, UpdateOutcome.DECLINED)
|
||||||
|
self.assertEqual(cache.get(item_doc, "k").value, 1)
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
self.assertEqual(cache.get(item_doc, "k").value, 5)
|
||||||
|
|
||||||
|
during: list[int] = []
|
||||||
|
|
||||||
|
def apply(item, calls):
|
||||||
|
during.append(cache.get(item_doc, "k").value)
|
||||||
|
if calls == 1:
|
||||||
|
rival.put(item_doc, "k", Item("n", 9))
|
||||||
|
return Item("n", item.value + 1)
|
||||||
|
|
||||||
|
cache.update(item_doc, "k", Counting(apply))
|
||||||
|
self.assertEqual(during, [5, 5]) # the lost attempt published nothing mid-update
|
||||||
|
|
||||||
|
def test_absent_key_rival_removal_and_a_store_that_holds_nothing(self) -> None:
|
||||||
|
a, b = self.cache(), self.cache()
|
||||||
|
bump = lambda item: Item("n", (item.value if item else 0) + 1) # noqa: E731
|
||||||
|
self.assertIs(a.update(item_doc, "k", bump).outcome, UpdateOutcome.UPDATED)
|
||||||
|
self.assertIs(b.update(item_doc, "k", bump).outcome, UpdateOutcome.UPDATED)
|
||||||
|
self.assertEqual(self.stored().value, 2)
|
||||||
|
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "d", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
apply = Counting(lambda item, calls: (rival.delete(item_doc, "d") if calls == 1 else None) or Item("n", (item.value if item else 0) + 1))
|
||||||
|
cache.update(item_doc, "d", apply)
|
||||||
|
self.assertEqual(apply.seen, [1, None])
|
||||||
|
self.assertEqual(self.stored("d").value, 1)
|
||||||
|
|
||||||
|
holder = self.cache()
|
||||||
|
holder.put(item_doc, "h", Item("n", 3))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
rival.delete(item_doc, "h")
|
||||||
|
nothing = Counting(lambda item, calls: Item("n", 1))
|
||||||
|
holder.update(item_doc, "h", nothing)
|
||||||
|
self.assertEqual(nothing.seen, [None])
|
||||||
|
|
||||||
|
def test_a_rival_writing_and_restoring_the_value_still_reruns_apply(self) -> None:
|
||||||
|
cache, rival = self.cache(), self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
rival.pull_all_backing_stores()
|
||||||
|
|
||||||
|
def apply(item, calls):
|
||||||
|
if calls == 1:
|
||||||
|
rival.put(item_doc, "k", Item("n", 5))
|
||||||
|
rival.put(item_doc, "k", Item("n", 1))
|
||||||
|
return Item("n", item.value + 1)
|
||||||
|
|
||||||
|
counting = Counting(apply)
|
||||||
|
cache.update(item_doc, "k", counting)
|
||||||
|
self.assertEqual(counting.seen, [1, 1])
|
||||||
|
self.assertEqual(self.stored().value, 2)
|
||||||
|
|
||||||
|
def test_a_stale_caches_first_attempt_reads_the_store(self) -> None:
|
||||||
|
a, b = self.cache(), self.cache()
|
||||||
|
a.put(item_doc, "k", Item("n", 1))
|
||||||
|
b.pull_all_backing_stores()
|
||||||
|
a.put(item_doc, "k", Item("n", 7))
|
||||||
|
apply = Counting(lambda item, calls: Item("n", item.value + 1))
|
||||||
|
b.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.seen, [7])
|
||||||
|
self.assertEqual(self.stored().value, 8)
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateRefusalTests(Fixture):
|
||||||
|
def refused(self, cache: CultCache, key: str = "k", document=item_doc, *errors) -> None:
|
||||||
|
before = self.bytes()
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaises(errors or (CultCacheError, SchemaConflictError)):
|
||||||
|
cache.update(document, key, apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
|
||||||
|
def test_refusals_before_apply(self) -> None:
|
||||||
|
writer = self.cache(item_doc, other_doc)
|
||||||
|
writer.put(other_doc, "k", Item("n", 1))
|
||||||
|
self.refused(writer) # the record at the key is another document's, held
|
||||||
|
self.refused(self.fresh(item_doc, other_doc)) # unpulled: the store's record is another document's
|
||||||
|
self.refused(self.cache(item_doc)) # foreign: the store holds a type this cache never registered
|
||||||
|
self.refused(self.fresh(item_doc))
|
||||||
|
storeless = self.cache(item_doc, other_doc, store=False)
|
||||||
|
storeless.put(other_doc, "k", Item("n", 1))
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaises((CultCacheError, SchemaConflictError)):
|
||||||
|
storeless.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
|
||||||
|
def fresh(self, *documents) -> CultCache:
|
||||||
|
"""A cache that has not pulled: whatever it reads, it reads from the store at update."""
|
||||||
|
builder = CultCache.builder()
|
||||||
|
for document in documents:
|
||||||
|
builder.register_document_type(document)
|
||||||
|
return builder.add_generic_store(SingleFileMessagePackBackingStore(self.path)).build()
|
||||||
|
|
||||||
|
def test_a_home_without_read_and_commit_if_refuses(self) -> None:
|
||||||
|
jsonl = JsonLinesBackingStore(self.dir / "state.jsonl")
|
||||||
|
cache = CultCache.builder().register_document_type(item_doc).add_generic_store(jsonl).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
before = (self.dir / "state.jsonl").read_bytes()
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "cannot write conditionally"):
|
||||||
|
cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
self.assertEqual((self.dir / "state.jsonl").read_bytes(), before)
|
||||||
|
self.assertFalse(hasattr(JsonLinesBackingStore, "read") or hasattr(JsonLinesBackingStore, "commit_if"))
|
||||||
|
|
||||||
|
class Bare:
|
||||||
|
def pull_all(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def push(self, envelope):
|
||||||
|
raise AssertionError("update wrote through a store that cannot write conditionally")
|
||||||
|
|
||||||
|
def push_all(self, envelopes, soft=False):
|
||||||
|
raise AssertionError("push_all")
|
||||||
|
|
||||||
|
def delete(self, type, key):
|
||||||
|
raise AssertionError("delete")
|
||||||
|
|
||||||
|
bare = CultCache.builder().register_document_type(item_doc).add_generic_store(Bare()).build()
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "Bare"):
|
||||||
|
bare.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
|
||||||
|
def test_read_only_records_refuse_before_apply(self) -> None:
|
||||||
|
wide = define_database_entry_type(
|
||||||
|
"tests.rw",
|
||||||
|
[database_entry_field("name", 0, codec=t.string), database_entry_field("extra", 1, codec=t.string)],
|
||||||
|
cls=Wide,
|
||||||
|
schema_id="tests.rw.v1",
|
||||||
|
)
|
||||||
|
narrow = define_database_entry_type("tests.rw", [database_entry_field("name", 0, codec=t.string)], cls=Narrow, schema_id="tests.rw.v1")
|
||||||
|
self.cache(wide).put(wide, "k", Wide("n", "x"))
|
||||||
|
before = self.bytes()
|
||||||
|
for pulled in (True, False): # held read-only, then a store record the cache never loaded
|
||||||
|
with self.subTest(pulled=pulled):
|
||||||
|
cache = self.cache(narrow) if pulled else self.fresh(narrow)
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaises(SchemaConflictError) as caught:
|
||||||
|
cache.update(narrow, "k", apply)
|
||||||
|
self.assertEqual(caught.exception.record_key, "k")
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
|
||||||
|
def test_a_non_global_key_for_a_global_document_refuses(self) -> None:
|
||||||
|
cache = self.cache(global_doc)
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaises(CultCacheError):
|
||||||
|
cache.update(global_doc, "not-global", apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
|
||||||
|
def test_refusals_after_apply_are_not_retried(self) -> None:
|
||||||
|
cache = self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
before = self.bytes()
|
||||||
|
boom = Counting(lambda item, calls: (_ for _ in ()).throw(RuntimeError("boom")))
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "boom"):
|
||||||
|
cache.update(item_doc, "k", boom)
|
||||||
|
self.assertEqual(boom.calls, 1)
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
|
||||||
|
indexed = self.cache(sku_doc, path=self.dir / "sku.cc")
|
||||||
|
indexed.put(sku_doc, "a", Item("n", 1, "X"))
|
||||||
|
indexed.put(sku_doc, "b", Item("n", 1, "Y"))
|
||||||
|
before = (self.dir / "sku.cc").read_bytes()
|
||||||
|
clash = Counting(lambda item, calls: Item("n", 2, "X"))
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "unique index"):
|
||||||
|
indexed.update(sku_doc, "b", clash)
|
||||||
|
self.assertEqual(clash.calls, 1)
|
||||||
|
self.assertEqual((self.dir / "sku.cc").read_bytes(), before)
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateOwnWritesTests(Fixture):
|
||||||
|
def test_own_writes_and_the_held_instance(self) -> None:
|
||||||
|
cache = self.cache()
|
||||||
|
cache.put(item_doc, "k", Item("n", 1))
|
||||||
|
apply = Counting(lambda item, calls: Item("n", item.value + 1))
|
||||||
|
cache.update(item_doc, "k", apply)
|
||||||
|
second = cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.calls, 2)
|
||||||
|
self.assertIs(second.document, cache.get(item_doc, "k"))
|
||||||
|
self.assertEqual(second.document.value, 3)
|
||||||
|
|
||||||
|
def sideways(item, calls):
|
||||||
|
cache.put(item_doc, "side", Item("n", 5)) # apply runs outside the cache's hold
|
||||||
|
return Item("n", item.value + 1)
|
||||||
|
|
||||||
|
sideways_apply = Counting(sideways)
|
||||||
|
result = cache.update(item_doc, "k", sideways_apply)
|
||||||
|
self.assertEqual((result.outcome, sideways_apply.calls), (UpdateOutcome.UPDATED, 1))
|
||||||
|
self.assertEqual(self.stored("side").value, 5)
|
||||||
|
|
||||||
|
def test_store_less_cache(self) -> None:
|
||||||
|
cache = self.cache(store=False)
|
||||||
|
cache.put(item_doc, "k", Item("n", 0))
|
||||||
|
results: list[UpdateResult] = []
|
||||||
|
|
||||||
|
def bump(item):
|
||||||
|
time.sleep(0)
|
||||||
|
return Item("n", item.value + 1)
|
||||||
|
|
||||||
|
def run() -> None:
|
||||||
|
for _ in range(50):
|
||||||
|
results.append(cache.update(item_doc, "k", bump, max_attempts=500))
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=run) for _ in range(2)]
|
||||||
|
for thread in threads:
|
||||||
|
thread.start()
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
self.assertTrue(all(result.outcome is UpdateOutcome.UPDATED for result in results))
|
||||||
|
self.assertEqual(cache.get(item_doc, "k").value, 100)
|
||||||
|
|
||||||
|
# A store that becomes the type's home while apply runs holds a record the cache never read: the commit loses.
|
||||||
|
cache = self.cache(store=False) # holds no record of the type, so a store may become its home
|
||||||
|
other = self.cache(path=self.dir / "late.cc")
|
||||||
|
other.put(item_doc, "fresh", Item("n", 40))
|
||||||
|
late = SingleFileMessagePackBackingStore(self.dir / "late.cc")
|
||||||
|
|
||||||
|
def attach(item, calls):
|
||||||
|
if calls == 1:
|
||||||
|
cache.add_generic_store(late)
|
||||||
|
return Item("n", (item.value if item else 0) + 1)
|
||||||
|
|
||||||
|
apply = Counting(attach)
|
||||||
|
result = cache.update(item_doc, "fresh", apply)
|
||||||
|
self.assertEqual(apply.seen, [None, 40])
|
||||||
|
self.assertIs(result.outcome, UpdateOutcome.UPDATED)
|
||||||
|
self.assertEqual(self.cache(path=self.dir / "late.cc").get(item_doc, "fresh").value, 41)
|
||||||
|
|
||||||
|
def test_a_home_attached_while_apply_runs_is_reread_or_refused(self) -> None:
|
||||||
|
"""The cache's home for the type changes between the read and the commit (attaching is allowed while it holds no record
|
||||||
|
of the type): the version read from the old home is no condition on the new one."""
|
||||||
|
cache = self.cache(path=self.dir / "generic.cc")
|
||||||
|
late = SingleFileMessagePackBackingStore(self.dir / "late.cc")
|
||||||
|
|
||||||
|
def attach(item, calls):
|
||||||
|
if calls == 1:
|
||||||
|
cache.add_backing_store(late, ["item"])
|
||||||
|
return Item("n", 1)
|
||||||
|
|
||||||
|
apply = Counting(attach)
|
||||||
|
self.assertIs(cache.update(item_doc, "k", apply).outcome, UpdateOutcome.UPDATED)
|
||||||
|
self.assertEqual(apply.seen, [None, None]) # reread from the new home, not committed with the old home's version
|
||||||
|
self.assertEqual(self.cache(path=self.dir / "late.cc").get(item_doc, "k").value, 1)
|
||||||
|
|
||||||
|
cache = self.cache(path=self.dir / "generic2.cc")
|
||||||
|
jsonl = JsonLinesBackingStore(self.dir / "late.jsonl")
|
||||||
|
|
||||||
|
def attach_jsonl(item, calls):
|
||||||
|
cache.add_backing_store(jsonl, ["item"])
|
||||||
|
return Item("n", 1)
|
||||||
|
|
||||||
|
apply = Counting(attach_jsonl)
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "cannot write conditionally"):
|
||||||
|
cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.calls, 1)
|
||||||
|
self.assertFalse((self.dir / "late.jsonl").exists())
|
||||||
|
|
||||||
|
def test_ledger_under_contention(self) -> None:
|
||||||
|
caches = [self.cache() for _ in range(3)]
|
||||||
|
outcomes: list[UpdateOutcome] = []
|
||||||
|
|
||||||
|
def run(cache) -> None:
|
||||||
|
for _ in range(25):
|
||||||
|
outcomes.append(cache.update(item_doc, "k", lambda item: Item("n", (item.value if item else 0) + 1)).outcome)
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=run, args=(cache,)) for cache in caches]
|
||||||
|
for thread in threads:
|
||||||
|
thread.start()
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
self.assertEqual(self.stored().value, outcomes.count(UpdateOutcome.UPDATED))
|
||||||
|
self.assertEqual(len(outcomes), 75)
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateLegacyGlobalTests(Fixture):
|
||||||
|
def plant(self, key: str = "legacy") -> None:
|
||||||
|
entry = global_doc.catalog_entry()
|
||||||
|
SingleFileMessagePackBackingStore(self.path).push(
|
||||||
|
CultCacheEnvelope.create(key=key, type=global_doc.type, payload=global_doc.encode_payload(Item("n", 1)), schema_id=entry.schema_id, catalog_entry=entry)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_update_global_updates_the_global_record(self) -> None:
|
||||||
|
cache = self.cache(global_doc)
|
||||||
|
result = cache.update_global(global_doc, lambda item: Item("n", (item.value if item else 0) + 1))
|
||||||
|
self.assertIs(result.outcome, UpdateOutcome.UPDATED)
|
||||||
|
self.assertEqual(self.cache(global_doc).get(global_doc, GLOBAL_KEY).value, 1)
|
||||||
|
|
||||||
|
def test_a_legacy_global_is_refused_until_put_global_retires_it(self) -> None:
|
||||||
|
self.plant()
|
||||||
|
cache = self.cache(global_doc) # loaded from the legacy key
|
||||||
|
before = self.bytes()
|
||||||
|
for call in (lambda a: cache.update_global(global_doc, a), lambda a: cache.update(global_doc, GLOBAL_KEY, a)):
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "put_global"):
|
||||||
|
call(apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
# Judged from the cache's load, not only from the store: the legacy record is gone from the store, the cache still holds it.
|
||||||
|
SingleFileMessagePackBackingStore(self.path).delete(global_doc.type, "legacy")
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "put_global"):
|
||||||
|
cache.update_global(global_doc, apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
cache.put_global(global_doc, Item("n", 5))
|
||||||
|
self.assertEqual(cache.update_global(global_doc, lambda item: Item("n", item.value + 1)).document.value, 6)
|
||||||
|
|
||||||
|
def test_a_pull_inside_apply_that_loads_a_legacy_global_fails_the_commit(self) -> None:
|
||||||
|
cache = self.cache(global_doc)
|
||||||
|
|
||||||
|
def apply(item):
|
||||||
|
self.plant()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
return Item("n", 9)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "put_global"):
|
||||||
|
cache.update_global(global_doc, apply)
|
||||||
|
self.assertEqual([envelope.key for envelope in SingleFileMessagePackBackingStore(self.path).pull_all()], ["legacy"])
|
||||||
|
|
||||||
|
def test_an_unpulled_cache_judges_the_legacy_global_from_the_store(self) -> None:
|
||||||
|
self.plant()
|
||||||
|
before = self.bytes()
|
||||||
|
cache = SingleFileMessagePackBackingStore(self.path)
|
||||||
|
unpulled = CultCache.builder().register_document_type(global_doc).add_generic_store(cache).build()
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaisesRegex(CultCacheError, "put_global"):
|
||||||
|
unpulled.update_global(global_doc, apply)
|
||||||
|
self.assertEqual(apply.calls, 0)
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
|
||||||
|
|
||||||
|
class StoreRestampTests(Fixture):
|
||||||
|
def envelope(self, value: int, stored_at: str = ALPHA, key: str = "k") -> CultCacheEnvelope:
|
||||||
|
entry = item_doc.catalog_entry()
|
||||||
|
return CultCacheEnvelope(key, "item", item_doc.encode_payload(Item("n", value)), stored_at, entry.schema_id, entry)
|
||||||
|
|
||||||
|
def stamp_on_disk(self, store, key: str = "k") -> str:
|
||||||
|
return next(envelope.stored_at for envelope in store.pull_all() if envelope.key == key)
|
||||||
|
|
||||||
|
def later_than_alpha(self, text: str) -> bool:
|
||||||
|
return parse_stored_at(text) > parse_stored_at(ALPHA)
|
||||||
|
|
||||||
|
def test_commit_if_and_the_restamp_at_the_store(self) -> None:
|
||||||
|
store = SingleFileMessagePackBackingStore(self.path)
|
||||||
|
self.assertIsNone(store.read("k"))
|
||||||
|
stamp = store.commit_if(self.envelope(1), None) # None against an absent key commits
|
||||||
|
self.assertEqual(stamp, ALPHA)
|
||||||
|
envelope, version = store.read("k")
|
||||||
|
self.assertEqual(version, record_version(envelope))
|
||||||
|
before = self.bytes()
|
||||||
|
self.assertIsNone(store.commit_if(self.envelope(2, "2100-01-01T00:00:00Z"), None)) # a record is there: not absent
|
||||||
|
self.assertIsNone(store.commit_if(self.envelope(2, "2100-01-01T00:00:00Z"), "0" * 64))
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
written = store.commit_if(self.envelope(2), version) # a staged stamp that is no later is minted over
|
||||||
|
self.assertTrue(self.later_than_alpha(written))
|
||||||
|
self.assertEqual(self.stamp_on_disk(store), written)
|
||||||
|
self.assertEqual(stamp_of(store, "k"), written)
|
||||||
|
|
||||||
|
def test_every_replacing_write_is_stored_strictly_later(self) -> None:
|
||||||
|
for make in (SingleFileMessagePackBackingStore, JsonLinesBackingStore):
|
||||||
|
for name in ("push", "push_all", "commit_if"):
|
||||||
|
if name == "commit_if" and make is JsonLinesBackingStore:
|
||||||
|
continue
|
||||||
|
with self.subTest(store=make.__name__, write=name):
|
||||||
|
path = self.dir / f"{make.__name__}-{name}"
|
||||||
|
store = make(path)
|
||||||
|
store.push(self.envelope(1))
|
||||||
|
if name == "push":
|
||||||
|
reported = store.push(self.envelope(2))
|
||||||
|
elif name == "push_all":
|
||||||
|
reported = store.push_all([self.envelope(2)])["k"]
|
||||||
|
else:
|
||||||
|
reported = store.commit_if(self.envelope(2), store.read("k")[1])
|
||||||
|
self.assertTrue(self.later_than_alpha(reported))
|
||||||
|
self.assertEqual(self.stamp_on_disk(store), reported)
|
||||||
|
|
||||||
|
def test_the_cache_holds_the_stamp_the_store_wrote(self) -> None:
|
||||||
|
for name in ("put", "put_envelope", "put_envelopes", "update"):
|
||||||
|
with self.subTest(write=name):
|
||||||
|
path = self.dir / name
|
||||||
|
SingleFileMessagePackBackingStore(path).push(self.envelope(1))
|
||||||
|
cache = self.cache(path=path)
|
||||||
|
self.assertEqual(cache.get_envelope(item_doc, "k").stored_at, ALPHA)
|
||||||
|
if name == "put":
|
||||||
|
cache.put(item_doc, "k", Item("n", 2))
|
||||||
|
elif name == "put_envelope":
|
||||||
|
cache.put_envelope(item_doc, self.envelope(2))
|
||||||
|
elif name == "put_envelopes":
|
||||||
|
cache.put_envelopes(item_doc, [self.envelope(2)])
|
||||||
|
else:
|
||||||
|
cache.update(item_doc, "k", lambda item: Item("n", 2))
|
||||||
|
held = cache.get_envelope(item_doc, "k").stored_at
|
||||||
|
self.assertTrue(self.later_than_alpha(held))
|
||||||
|
self.assertEqual(held, self.stamp_on_disk(SingleFileMessagePackBackingStore(path)))
|
||||||
|
|
||||||
|
def test_a_store_less_cache_stamps_a_replacing_write_later(self) -> None:
|
||||||
|
cache = self.cache(store=False)
|
||||||
|
cache.put_envelope(item_doc, self.envelope(1))
|
||||||
|
cache.put_envelope(item_doc, self.envelope(2, "2000-01-01T00:00:00Z"))
|
||||||
|
self.assertTrue(self.later_than_alpha(cache.get_envelope(item_doc, "k").stored_at))
|
||||||
|
|
||||||
|
def test_only_the_stamp_shape_the_runtimes_write_parses(self) -> None:
|
||||||
|
"""An RFC 3339 date-time with T, a fraction of 1 to 9 digits and Z or a numeric offset, on every interpreter (the shape is
|
||||||
|
checked here, not by fromisoformat), so a stamp reads the same on 3.10 and 3.13. The edges follow RFC 3339: offset hours 00-23,
|
||||||
|
minutes 00-59."""
|
||||||
|
utc = timezone.utc
|
||||||
|
accepted = {
|
||||||
|
"2026-01-01T00:00:00Z": datetime(2026, 1, 1, tzinfo=utc),
|
||||||
|
"2026-01-01T00:00:00.1Z": datetime(2026, 1, 1, 0, 0, 0, 100000, utc),
|
||||||
|
"2026-01-01T00:00:00.123456789Z": datetime(2026, 1, 1, 0, 0, 0, 123456, utc),
|
||||||
|
"2026-01-01T00:00:00.1234567+00:00": datetime(2026, 1, 1, 0, 0, 0, 123456, utc),
|
||||||
|
"2026-01-01T05:30:00+05:30": datetime(2026, 1, 1, tzinfo=utc),
|
||||||
|
"2025-12-31T19:00:00-05:00": datetime(2026, 1, 1, tzinfo=utc),
|
||||||
|
"2026-01-01T23:59:00+23:59": datetime(2026, 1, 1, 0, 0, tzinfo=utc),
|
||||||
|
}
|
||||||
|
for text, instant in accepted.items():
|
||||||
|
with self.subTest(text=text):
|
||||||
|
self.assertEqual(parse_stored_at(text), instant)
|
||||||
|
for text in (
|
||||||
|
"2026-01-01", "20260101T000000Z", "2026-W01-1T00:00:00Z", "2026-01-01 00:00:00Z", "2026-01-01T00:00:00,5Z",
|
||||||
|
"2026-01-01T00Z", "2026-01-01T00:00:00", "2026-01-01T00:00:00.Z", "2026-01-01T00:00:00.1234567890Z",
|
||||||
|
"2026-01-01T00:00:00z", " 2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z ", "2026-01-01T24:00:00Z",
|
||||||
|
"2026-13-01T00:00:00Z", "2026-01-01T00:00:00+24:00", "2026-01-01T00:00:00+05:60", "2026-01-01T00:00:00+05:75", "2026-01-01T00:00Z", "2026-01-01T00:00+00:00",
|
||||||
|
"026-01-01T00:00:00Z", "12026-01-01T00:00:00Z", "2026-01-01T00:00:00+5:00", "2026-01-01T00:00:00+0000", "\u0662026-01-01T00:00:00Z",
|
||||||
|
"", "garbage",
|
||||||
|
):
|
||||||
|
with self.subTest(text=text):
|
||||||
|
self.assertIsNone(parse_stored_at(text))
|
||||||
|
|
||||||
|
def test_a_stamp_outside_the_range_is_unparseable_and_one_at_the_edge_refuses_a_later_stamp(self) -> None:
|
||||||
|
for text in ("9999-12-31T23:30:00-05:00", "0001-01-01T00:00:00+05:00"):
|
||||||
|
with self.subTest(text=text):
|
||||||
|
self.assertIsNone(parse_stored_at(text))
|
||||||
|
now = datetime(2030, 1, 1, tzinfo=timezone.utc)
|
||||||
|
self.assertEqual(mint_stored_at("9999-12-31T23:30:00-05:00", now), "2030-01-01T00:00:00.000000+00:00")
|
||||||
|
edge = "9999-12-31T23:59:59.9999999Z"
|
||||||
|
with self.assertRaisesRegex(ValueError, "last instant"):
|
||||||
|
mint_stored_at(edge, now)
|
||||||
|
with self.assertRaisesRegex(ValueError, "last instant"):
|
||||||
|
stored_at_over("2000-01-01T00:00:00Z", edge)
|
||||||
|
# A write over a record stamped at the edge is refused by the store and writes nothing, from push, commit_if and update.
|
||||||
|
store = SingleFileMessagePackBackingStore(self.path)
|
||||||
|
entry = item_doc.catalog_entry()
|
||||||
|
record = lambda value, at: CultCacheEnvelope("k", "item", item_doc.encode_payload(Item("n", value)), at, entry.schema_id, entry) # noqa: E731
|
||||||
|
store.push(record(1, edge))
|
||||||
|
before = self.bytes()
|
||||||
|
version = store.read("k")[1]
|
||||||
|
for write in (lambda: store.push(record(2, ALPHA)), lambda: store.push_all([record(2, ALPHA)]), lambda: store.commit_if(record(2, ALPHA), version)):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
write()
|
||||||
|
cache = self.cache()
|
||||||
|
apply = Counting()
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
cache.update(item_doc, "k", apply)
|
||||||
|
self.assertEqual(apply.calls, 1)
|
||||||
|
self.assertEqual(self.bytes(), before)
|
||||||
|
self.assertEqual(cache.get_envelope(item_doc, "k").stored_at, edge)
|
||||||
|
|
||||||
|
def test_stamp_parsing_and_install(self) -> None:
|
||||||
|
for text in ("2026-01-01T00:00:00.1234567+00:00", "2026-01-01T00:00:00Z", "2026-01-01T00:00:00.123456+00:00"):
|
||||||
|
with self.subTest(text=text):
|
||||||
|
self.assertIsNotNone(parse_stored_at(text))
|
||||||
|
self.assertEqual(parse_stored_at(text).tzinfo, timezone.utc)
|
||||||
|
self.assertIsNone(parse_stored_at("garbage"))
|
||||||
|
self.assertIsNone(parse_stored_at(None))
|
||||||
|
now = datetime(2030, 1, 1, tzinfo=timezone.utc)
|
||||||
|
self.assertEqual(mint_stored_at("garbage", now), "2030-01-01T00:00:00.000000+00:00")
|
||||||
|
self.assertEqual(stored_at_over("2000-01-01T00:00:00Z", "2000-01-01T00:00:00Z")[:4], str(datetime.now(timezone.utc).year)) # no later: minted at now
|
||||||
|
self.assertEqual(mint_stored_at("2030-01-01T00:00:00Z", now), "2030-01-01T00:00:00.000001+00:00") # equal is not later
|
||||||
|
self.assertEqual(mint_stored_at("2099-01-01T00:00:00.1234567Z", now), "2099-01-01T00:00:00.123457+00:00")
|
||||||
|
self.assertEqual(stored_at_over("2000-01-01T00:00:00Z", "2099-01-01T00:00:00.1234567Z"), "2099-01-01T00:00:00.123457+00:00")
|
||||||
|
|
||||||
|
for reported in (True, 1, "garbage"):
|
||||||
|
with self.subTest(reported=reported):
|
||||||
|
class Lying(SingleFileMessagePackBackingStore):
|
||||||
|
def push(self, envelope):
|
||||||
|
super().push(envelope)
|
||||||
|
return reported
|
||||||
|
|
||||||
|
cache = CultCache.builder().register_document_type(item_doc).add_generic_store(Lying(self.dir / f"lying-{reported}")).build()
|
||||||
|
cache.pull_all_backing_stores()
|
||||||
|
cache.put_envelope(item_doc, self.envelope(1, "2026-02-02T00:00:00.000000+00:00"))
|
||||||
|
self.assertEqual(cache.get_envelope(item_doc, "k").stored_at, "2026-02-02T00:00:00.000000+00:00")
|
||||||
|
|
||||||
|
|
||||||
|
def stamp_of(store, key: str) -> str:
|
||||||
|
return store.read(key)[0].stored_at
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
1
packages/cultcache-rs/.gitignore
vendored
1
packages/cultcache-rs/.gitignore
vendored
|
|
@ -1,2 +1 @@
|
||||||
/target
|
/target
|
||||||
Cargo.lock
|
|
||||||
|
|
|
||||||
|
|
@ -6,14 +6,24 @@ only in git history.
|
||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- `CultCache::update_with_attempts` and the conditional `update`: each attempt reads the record from its home store with its content version in one read, applies a function to a copy and commits only over that version, rerunning on a lost race (default five attempts). The result is `CultUpdateResult<T>`, outcome `Updated`, `Declined` (the function returned `None`) or `LostRace`, with the cache's document. Public types `CultStoredRecord`, `CultCommitIf`, `CultUpdateOutcome`, `CultUpdateResult`.
|
||||||
|
- `CacheBackingStore::read(key)` and `commit_if(entry, expected)`, provided methods whose defaults refuse with an error naming the store kind; `SingleFileMessagePackBackingStore` implements both. The content version is the lowercase hex SHA-256 of the record re-encoded as the store file writes it (key, schema id, `stored_at`, payload). It equals the SHA-256 of the record's bytes in the file, and so the C# reference's `tests/vectors/content-version`, whenever the file's MessagePack encodings are minimal, which every runtime's writer produces; a hand-edited file with a non-minimal encoding (for example a `str8` key that fits a fixstr) hashes differently from its slice here. New dependency `sha2` 0.10 and eight new lockfile crates: `sha2`, `digest`, `block-buffer`, `crypto-common`, `generic-array`, `typenum`, `cpufeatures`, `version_check`.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
- `is_stored_at_stamp(&str)` (new, public) is the one shape a peer's `storedAt` must parse as, pinned by `tests/vectors/stored-at-stamp` (RFC 3339 `YYYY-MM-DDThh:mm:ss`, a 1 to 9 digit fraction, `Z` or `+hh:mm`/`-hh:mm`; uppercase `T`/`Z`, a real date, seconds 00-59, years 0001-9999), hand-checked because `chrono`'s RFC 3339 reader takes other shapes. `put_raw_envelope` and `put_prepared_batch` refuse a `stored_at` that is not a stamp (before: only a blank one) and mint nothing (variants:ruling:raw-put-without-stamp-refused).
|
||||||
|
- The single-file store's `push`, `push_all` and `commit_if` stamp a replacing write strictly later than the record it replaces at the same key (a staged stamp stands only when it parses and is strictly later; otherwise the stamp is minted as the replaced stamp plus one microsecond, or now if later), with no identical-content exception, as C# does. A stamp with no successor (the replaced stamp is at year 9999's last microsecond) refuses the write and writes nothing, as C# `MintStoredAt` does. `commit_if` replaces only the record of the entry's own type under the key, so a retype is refused as `push` refuses it. The cache holds the stamp the store wrote. The redb stores and the store-level exact-envelope writes (`compare_and_swap_entry` and its family) still write the caller's envelope unchanged.
|
||||||
|
- The single-file store re-checks for a link once the store lock is held (every write and the try-exchange), so a link put at the path while a writer waited on the lock is refused with `StoreLinkedError` and the lock released, as C# does, instead of being replaced. Pinned by a test that puts the link while a second holder owns the lock.
|
||||||
- `CultCache::is_writable` does not look at store paths: a link at a store path is not predicted, and the write refuses it with `StoreLinkedError`. Pinned by a test through a cache attached to a linked store.
|
- `CultCache::is_writable` does not look at store paths: a link at a store path is not predicted, and the write refuses it with `StoreLinkedError`. Pinned by a test through a cache attached to a linked store.
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0).
|
Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0).
|
||||||
|
|
||||||
|
- `CultCache::update` is replaced, not extended: `update::<T, _>(key, apply)` takes `FnMut(Option<T>) -> Option<T>` and returns `Result<CultUpdateResult<T>>`. The blind `FnOnce(Option<T>) -> T` form that returned `T` is gone; use `put` for a blind write. It reads the home store, not the cache's image, so a cache whose store has no `read` and `commit_if` (the redb stores, a custom store) refuses it before `apply` runs.
|
||||||
|
- `CacheBackingStore::push` returns `Result<String>`, the `stored_at` written, and `push_all` returns `Result<Vec<String>>`, one stamp per entry given in order (a soft write that wrote nothing returns an empty vec). A custom store must return what it wrote; the redb stores return the entries' own stamps. Huginn implements `CacheBackingStore` in five test stores under a pinned git rev: they change at its next bump.
|
||||||
- A write at a store path that is a symbolic link, live or dangling, is refused with the new public `StoreLinkedError` (`error.downcast_ref::<StoreLinkedError>()`) before the lock and the replace, and writes nothing: `push`, `push_all`, `delete`, the exchanges, and a soft `push_all`, which used to skip a link and now refuses. Before, a write at a live link replaced the link with a regular file and a dangling link failed with `io::ErrorKind::NotFound`. The redb stores (`RedbMessagePackBackingStore`, `OwnedRedbMessagePackBackingStore`) are exempt: they are ruled migrated and deleted (`paged-store-redb-migrate-and-delete`) and still open a live link's target. Configure the link's target as the store path. Reads through a live link work.
|
- A write at a store path that is a symbolic link, live or dangling, is refused with the new public `StoreLinkedError` (`error.downcast_ref::<StoreLinkedError>()`) before the lock and the replace, and writes nothing: `push`, `push_all`, `delete`, the exchanges, and a soft `push_all`, which used to skip a link and now refuses. Before, a write at a live link replaced the link with a regular file and a dangling link failed with `io::ErrorKind::NotFound`. The redb stores (`RedbMessagePackBackingStore`, `OwnedRedbMessagePackBackingStore`) are exempt: they are ruled migrated and deleted (`paged-store-redb-migrate-and-delete`) and still open a live link's target. Configure the link's target as the store path. Reads through a live link work.
|
||||||
- A record whose payload carries unread slots is read-only. A payload with a slot past its type's declared slots, at any depth (record, nested record, list element, union arm, unit arm), still decodes, and every write or removal of its key is now refused before any store is touched: `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` (the whole batch) and `delete`, so `update` too. The refusal is `SchemaConflictError` with the new `unread_slots`; its message names the type and both slot counts and never a slot value. Holders of such a record used to overwrite it and drop the extra slots silently. The payload nesting refusal still runs first. A held payload this build cannot read at all (an arm it lacks from a newer writer, or a corrupt record) is read-only too: every write or removal is refused with an error naming the type and key (a newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, as no cache call removes it), and `is_writable` answers false. `load_soa` is a write of every held key it omits or changes, so it refuses to drop or replace a read-only record. `is_writable` also answers false for a key a store carries as a foreign record and for a type other than the one holding the key, as the writes do. Each write over a held key decodes the held payload once to ask (about 40% of a `put` of a 200,000-element record, measured in release); keys nothing holds pay nothing.
|
- A record whose payload carries unread slots is read-only. A payload with a slot past its type's declared slots, at any depth (record, nested record, list element, union arm, unit arm), still decodes, and every write or removal of its key is now refused before any store is touched: `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` (the whole batch) and `delete`, so `update` too. The refusal is `SchemaConflictError` with the new `unread_slots`; its message names the type and both slot counts and never a slot value. Holders of such a record used to overwrite it and drop the extra slots silently. The payload nesting refusal still runs first. A held payload this build cannot read at all (an arm it lacks from a newer writer, or a corrupt record) is read-only too: every write or removal is refused with an error naming the type and key (a newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, as no cache call removes it), and `is_writable` answers false. `load_soa` is a write of every held key it omits or changes, so it refuses to drop or replace a read-only record. `is_writable` also answers false for a key a store carries as a foreign record and for a type other than the one holding the key, as the writes do. Each write over a held key decodes the held payload once to ask (about 40% of a `put` of a 200,000-element record, measured in release); keys nothing holds pay nothing.
|
||||||
- cultmesh-rs `pull_rudp_catalog_snapshot` applies a snapshot as one cache commit (`put_prepared_batch`), as C# does: a snapshot document for a key the node holds read-only, or any other refusal, applies nothing. It used to apply document by document, so a refusal ended the pull with the earlier documents kept. A snapshot whose types route to different backing stores is now refused whole, as a batch cannot span stores. C# `CultCache.Commit` refuses a cross-store commit the same way (`The batch spans A and B; a commit lands in one home store`), so the two agree; a test pins it.
|
- cultmesh-rs `pull_rudp_catalog_snapshot` applies a snapshot as one cache commit (`put_prepared_batch`), as C# does: a snapshot document for a key the node holds read-only, or any other refusal, applies nothing. It used to apply document by document, so a refusal ended the pull with the earlier documents kept. A snapshot whose types route to different backing stores is now refused whole, as a batch cannot span stores. C# `CultCache.Commit` refuses a cross-store commit the same way (`The batch spans A and B; a commit lands in one home store`), so the two agree; a test pins it.
|
||||||
|
|
|
||||||
802
packages/cultcache-rs/Cargo.lock
generated
Normal file
802
packages/cultcache-rs/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,802 @@
|
||||||
|
# This file is automatically @generated by Cargo.
|
||||||
|
# It is not intended for manual editing.
|
||||||
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "android_system_properties"
|
||||||
|
version = "0.1.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anyhow"
|
||||||
|
version = "1.0.104"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ar_archive_writer"
|
||||||
|
version = "0.5.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "523e5e187fff36fcb8d1d19ccf0ca4ca43ec3d158316057f66dc6f72716bc5ec"
|
||||||
|
dependencies = [
|
||||||
|
"object",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "autocfg"
|
||||||
|
version = "1.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bitflags"
|
||||||
|
version = "2.13.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "block-buffer"
|
||||||
|
version = "0.10.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bumpalo"
|
||||||
|
version = "3.20.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cc"
|
||||||
|
version = "1.11.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155"
|
||||||
|
dependencies = [
|
||||||
|
"find-msvc-tools",
|
||||||
|
"shlex",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg-if"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chrono"
|
||||||
|
version = "0.4.45"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||||
|
dependencies = [
|
||||||
|
"iana-time-zone",
|
||||||
|
"js-sys",
|
||||||
|
"num-traits",
|
||||||
|
"serde",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation-sys"
|
||||||
|
version = "0.8.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cpufeatures"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crypto-common"
|
||||||
|
version = "0.1.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||||
|
dependencies = [
|
||||||
|
"generic-array",
|
||||||
|
"typenum",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cultcache-rs"
|
||||||
|
version = "0.3.1"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"chrono",
|
||||||
|
"cultcache-rs-derive",
|
||||||
|
"fs2",
|
||||||
|
"pretty_assertions",
|
||||||
|
"redb",
|
||||||
|
"rmp-serde",
|
||||||
|
"serde",
|
||||||
|
"serde_bytes",
|
||||||
|
"serde_json",
|
||||||
|
"serde_stacker",
|
||||||
|
"sha2",
|
||||||
|
"stacker",
|
||||||
|
"tempfile",
|
||||||
|
"uuid",
|
||||||
|
"windows-sys 0.59.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cultcache-rs-derive"
|
||||||
|
version = "0.2.0"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "diff"
|
||||||
|
version = "0.1.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "digest"
|
||||||
|
version = "0.10.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||||
|
dependencies = [
|
||||||
|
"block-buffer",
|
||||||
|
"crypto-common",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "errno"
|
||||||
|
version = "0.3.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fastrand"
|
||||||
|
version = "2.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "find-msvc-tools"
|
||||||
|
version = "0.1.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fs2"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-core"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-task"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-util"
|
||||||
|
version = "0.3.34"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"futures-task",
|
||||||
|
"pin-project-lite",
|
||||||
|
"slab",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "generic-array"
|
||||||
|
version = "0.14.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||||
|
dependencies = [
|
||||||
|
"typenum",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "getrandom"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"r-efi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone"
|
||||||
|
version = "0.1.65"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||||
|
dependencies = [
|
||||||
|
"android_system_properties",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"iana-time-zone-haiku",
|
||||||
|
"js-sys",
|
||||||
|
"log",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"windows-core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iana-time-zone-haiku"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "itoa"
|
||||||
|
version = "1.0.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "js-sys"
|
||||||
|
version = "0.3.107"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"futures-util",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libc"
|
||||||
|
version = "0.2.191"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "linux-raw-sys"
|
||||||
|
version = "0.12.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "log"
|
||||||
|
version = "0.4.35"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memchr"
|
||||||
|
version = "2.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-traits"
|
||||||
|
version = "0.2.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "object"
|
||||||
|
version = "0.39.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell"
|
||||||
|
version = "1.21.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pin-project-lite"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pretty_assertions"
|
||||||
|
version = "1.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d"
|
||||||
|
dependencies = [
|
||||||
|
"diff",
|
||||||
|
"yansi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro2"
|
||||||
|
version = "1.0.107"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "psm"
|
||||||
|
version = "0.1.32"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622"
|
||||||
|
dependencies = [
|
||||||
|
"ar_archive_writer",
|
||||||
|
"cc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quote"
|
||||||
|
version = "1.0.47"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "r-efi"
|
||||||
|
version = "6.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "redb"
|
||||||
|
version = "4.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8b317ef681e18f6091efbfab532eaf924a135c492d9d0299d0fbfae07e92a39b"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rmp"
|
||||||
|
version = "0.8.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rmp-serde"
|
||||||
|
version = "1.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155"
|
||||||
|
dependencies = [
|
||||||
|
"rmp",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustix"
|
||||||
|
version = "1.1.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"errno",
|
||||||
|
"libc",
|
||||||
|
"linux-raw-sys",
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustversion"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||||
|
dependencies = [
|
||||||
|
"serde_core",
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_bytes"
|
||||||
|
version = "0.11.19"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_core"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||||
|
dependencies = [
|
||||||
|
"serde_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_derive"
|
||||||
|
version = "1.0.229"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 3.0.10",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_json"
|
||||||
|
version = "1.0.154"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||||
|
dependencies = [
|
||||||
|
"itoa",
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"zmij",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "serde_stacker"
|
||||||
|
version = "0.1.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d4936375d50c4be7eff22293a9344f8e46f323ed2b3c243e52f89138d9bb0f4a"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
"serde_core",
|
||||||
|
"stacker",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sha2"
|
||||||
|
version = "0.10.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "shlex"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "slab"
|
||||||
|
version = "0.4.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "stacker"
|
||||||
|
version = "0.1.25"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"cfg-if",
|
||||||
|
"libc",
|
||||||
|
"psm",
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "2.0.119"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn"
|
||||||
|
version = "3.0.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tempfile"
|
||||||
|
version = "3.27.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||||
|
dependencies = [
|
||||||
|
"fastrand",
|
||||||
|
"getrandom",
|
||||||
|
"once_cell",
|
||||||
|
"rustix",
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "typenum"
|
||||||
|
version = "1.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-ident"
|
||||||
|
version = "1.0.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "uuid"
|
||||||
|
version = "1.29.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bed99139f7dac9b5cfe5f17be4d66dc7079a34dee1d74bcc77217119240e267c"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom",
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "version_check"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen"
|
||||||
|
version = "0.2.130"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"rustversion",
|
||||||
|
"wasm-bindgen-macro",
|
||||||
|
"wasm-bindgen-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-macro"
|
||||||
|
version = "0.2.130"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"
|
||||||
|
dependencies = [
|
||||||
|
"quote",
|
||||||
|
"wasm-bindgen-macro-support",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-macro-support"
|
||||||
|
version = "0.2.130"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"
|
||||||
|
dependencies = [
|
||||||
|
"bumpalo",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 3.0.10",
|
||||||
|
"wasm-bindgen-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-shared"
|
||||||
|
version = "0.2.130"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-ident",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi"
|
||||||
|
version = "0.3.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||||
|
dependencies = [
|
||||||
|
"winapi-i686-pc-windows-gnu",
|
||||||
|
"winapi-x86_64-pc-windows-gnu",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi-i686-pc-windows-gnu"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winapi-x86_64-pc-windows-gnu"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-core"
|
||||||
|
version = "0.62.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||||
|
dependencies = [
|
||||||
|
"windows-implement",
|
||||||
|
"windows-interface",
|
||||||
|
"windows-link",
|
||||||
|
"windows-result",
|
||||||
|
"windows-strings",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-implement"
|
||||||
|
version = "0.60.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-interface"
|
||||||
|
version = "0.59.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.119",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-link"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-result"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-strings"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-sys"
|
||||||
|
version = "0.59.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||||
|
dependencies = [
|
||||||
|
"windows-targets",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-sys"
|
||||||
|
version = "0.61.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-targets"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||||
|
dependencies = [
|
||||||
|
"windows_aarch64_gnullvm",
|
||||||
|
"windows_aarch64_msvc",
|
||||||
|
"windows_i686_gnu",
|
||||||
|
"windows_i686_gnullvm",
|
||||||
|
"windows_i686_msvc",
|
||||||
|
"windows_x86_64_gnu",
|
||||||
|
"windows_x86_64_gnullvm",
|
||||||
|
"windows_x86_64_msvc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_aarch64_gnullvm"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_aarch64_msvc"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_i686_gnu"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_i686_gnullvm"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_i686_msvc"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_x86_64_gnu"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_x86_64_gnullvm"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows_x86_64_msvc"
|
||||||
|
version = "0.52.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "yansi"
|
||||||
|
version = "1.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zmij"
|
||||||
|
version = "1.0.23"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
|
|
@ -23,6 +23,7 @@ serde = { version = "1", features = ["derive"] }
|
||||||
serde_bytes = "0.11"
|
serde_bytes = "0.11"
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
serde_stacker = "0.1.14"
|
serde_stacker = "0.1.14"
|
||||||
|
sha2 = "0.10"
|
||||||
stacker = "0.1.25"
|
stacker = "0.1.25"
|
||||||
uuid = { version = "1", features = ["v4"] }
|
uuid = { version = "1", features = ["v4"] }
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -186,7 +186,7 @@ without pretending Rust has runtime reflection hiding under the floorboards.
|
||||||
- `soa::<T>`
|
- `soa::<T>`
|
||||||
- `put::<T>`
|
- `put::<T>`
|
||||||
- `put_envelope::<T>`
|
- `put_envelope::<T>`
|
||||||
- `update::<T>`
|
- `update::<T>` and `update_with_attempts::<T>`: the conditional golden-path write (read the stored record, apply, commit only over what was read, rerun on a lost race)
|
||||||
- `delete::<T>`
|
- `delete::<T>`
|
||||||
- `snapshot`
|
- `snapshot`
|
||||||
- `snapshot_soa`
|
- `snapshot_soa`
|
||||||
|
|
@ -304,6 +304,14 @@ The contract, shared with every runtime, is one home store per document type
|
||||||
|
|
||||||
## Persistence Semantics
|
## Persistence Semantics
|
||||||
|
|
||||||
|
`update` is the golden path for changing a record another writer may also change: each attempt reads the record and its content
|
||||||
|
version from the home store, hands `apply` a copy, and commits only if the store still holds that version. `put` is the blind
|
||||||
|
write. The single-file store stamps a replacing write (`push`, `push_all` and `update`'s commit) strictly later than the record
|
||||||
|
it replaces, and the cache holds the stamp the store reports; the redb stores and the exact-envelope family
|
||||||
|
(`compare_and_swap_entry` and the other store-level conditional writes) write the caller's envelope byte for byte. A store
|
||||||
|
needs `read` and `commit_if` to serve `update`; the redb stores do not have them.
|
||||||
|
The version is the SHA-256 of the record re-encoded as the store file writes it; that equals the hash of the record's bytes in the file (and the C# reference's vector) whenever the file's MessagePack encodings are minimal, as every runtime's writer produces.
|
||||||
|
|
||||||
`put` persists before mutating the in-memory cache. If persistence fails, the
|
`put` persists before mutating the in-memory cache. If persistence fails, the
|
||||||
cache does not pretend the write succeeded.
|
cache does not pretend the write succeeded.
|
||||||
|
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
26
packages/cultcache-rs/tests/content_version_vector.rs
Normal file
26
packages/cultcache-rs/tests/content_version_vector.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
//! `tests/vectors/content-version`, written only by the C# reference: the content version of a stored record is the lowercase
|
||||||
|
//! hex SHA-256 of its persisted encoding (`variants:ruling:write-set-content-version-declared-ownership`). The store reports
|
||||||
|
//! the version of every record of the committed `store.cc` as `versions.msgpack` says.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use cultcache_rs::{CacheBackingStore, SingleFileMessagePackBackingStore};
|
||||||
|
|
||||||
|
fn vectors() -> PathBuf {
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/content-version")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_reports_every_version_the_reference_reads() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("store.cc");
|
||||||
|
std::fs::copy(vectors().join("store.cc"), &path).unwrap();
|
||||||
|
let rows: Vec<(String, String)> = rmp_serde::from_slice(&std::fs::read(vectors().join("versions.msgpack")).unwrap()).unwrap();
|
||||||
|
assert!(rows.len() >= 2);
|
||||||
|
let store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
for (key, version) in rows {
|
||||||
|
let record = store.read(&key).unwrap().unwrap_or_else(|| panic!("no record {key}"));
|
||||||
|
assert_eq!(record.version, version, "{key}");
|
||||||
|
}
|
||||||
|
assert_eq!(store.read("absent").unwrap(), None);
|
||||||
|
}
|
||||||
|
|
@ -1,7 +1,9 @@
|
||||||
//! The Rust CultCache first call from docs/start/README.md. The docs show the region between the
|
//! The Rust CultCache first call and update from docs/start/README.md. The docs show each region between its
|
||||||
//! `snippet` markers verbatim, and `docs_show_the_tested_snippet` fails when they differ.
|
//! `snippet` markers verbatim, and `docs_show_the_tested_snippets` fails when they differ.
|
||||||
|
|
||||||
use cultcache_rs::{CultCache, DatabaseEntry, SingleFileMessagePackBackingStore};
|
use std::path::Path;
|
||||||
|
|
||||||
|
use cultcache_rs::{CultCache, CultUpdateResult, DatabaseEntry, SingleFileMessagePackBackingStore};
|
||||||
|
|
||||||
// snippet:begin
|
// snippet:begin
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
|
@ -40,17 +42,59 @@ fn first_call() -> anyhow::Result<()> {
|
||||||
}
|
}
|
||||||
// snippet:end
|
// snippet:end
|
||||||
|
|
||||||
|
// snippet:update:begin
|
||||||
|
fn restock(path: &Path) -> anyhow::Result<CultUpdateResult<Item>> {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Item>()?;
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?;
|
||||||
|
cache.pull_all_backing_stores()?;
|
||||||
|
cache.update::<Item, _>("item:potion", |item| {
|
||||||
|
// The store's copy of the record, or nothing if it has none; this may run again on a lost race.
|
||||||
|
let mut item = item.unwrap_or(Item {
|
||||||
|
name: "Potion".to_string(),
|
||||||
|
category: "Consumable".to_string(),
|
||||||
|
value: 0,
|
||||||
|
});
|
||||||
|
item.value += 10;
|
||||||
|
Some(item)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// snippet:update:end
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn first_call_round_trips_through_a_second_cache() {
|
fn first_call_round_trips_through_a_second_cache() {
|
||||||
first_call().unwrap();
|
first_call().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn docs_show_the_tested_snippet() {
|
fn restock_twice_reads_twenty_through_a_second_cache() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("restock.cc");
|
||||||
|
assert_eq!(restock(&path).unwrap().outcome, cultcache_rs::CultUpdateOutcome::Updated);
|
||||||
|
let second = restock(&path).unwrap();
|
||||||
|
assert_eq!(second.outcome, cultcache_rs::CultUpdateOutcome::Updated);
|
||||||
|
assert_eq!(second.document.unwrap().value, 20);
|
||||||
|
|
||||||
|
let mut reopened = CultCache::new();
|
||||||
|
reopened.register_entry_type::<Item>().unwrap();
|
||||||
|
reopened.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
reopened.pull_all_backing_stores().unwrap();
|
||||||
|
assert_eq!(reopened.get_required::<Item>("item:potion").unwrap().value, 20);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The docs show the region between `// snippet:<region>begin` and `end` of this file as one fenced block.
|
||||||
|
fn docs_show_region(region: &str) {
|
||||||
let source = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/first_call.rs")).unwrap();
|
let source = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/first_call.rs")).unwrap();
|
||||||
let begin = source.find("// snippet:begin\n").unwrap() + "// snippet:begin\n".len();
|
let (begin_marker, end_marker) = (format!("// snippet:{region}begin\n"), format!("// snippet:{region}end\n"));
|
||||||
let end = source.find("// snippet:end\n").unwrap();
|
let begin = source.find(&begin_marker).unwrap() + begin_marker.len();
|
||||||
|
let end = source.find(&end_marker).unwrap();
|
||||||
let docs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../docs/start/README.md")).unwrap();
|
let docs = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/../../docs/start/README.md")).unwrap();
|
||||||
let block = format!("```rust\n{}```", &source[begin..end]);
|
let block = format!("```rust\n{}```", &source[begin..end]);
|
||||||
assert!(docs.contains(&block), "docs/start/README.md no longer shows tests/first_call.rs verbatim");
|
assert!(docs.contains(&block), "docs/start/README.md no longer shows the {region:?} region of tests/first_call.rs verbatim");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn docs_show_the_tested_snippets() {
|
||||||
|
docs_show_region("");
|
||||||
|
docs_show_region("update:");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
165
packages/cultcache-rs/tests/schema_catalog.rs
Normal file
165
packages/cultcache-rs/tests/schema_catalog.rs
Normal file
|
|
@ -0,0 +1,165 @@
|
||||||
|
//! A consumer of a single-file store reads the schema version a record was published under from the store's catalog, through the
|
||||||
|
//! public surface only: `pull_all`, `pull_schema_catalog` and `resolve_schema_entry`
|
||||||
|
//! (`cultlib-gaps:follow_up:rs-snapshot-schema-version`).
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use cultcache_rs::{
|
||||||
|
CacheBackingStore, CultCache, DatabaseEntry, SchemaCatalogEntry, SingleFileMessagePackBackingStore, resolve_schema_entry,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "tests.sc_one", schema = "tests.sc_one")]
|
||||||
|
struct One {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "tests.sc_two", schema = "tests.sc_two")]
|
||||||
|
struct Two {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vectors(folder: &str) -> PathBuf {
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors").join(folder)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The version a store publishes for each record, in key order: `(key, schema name, schema version)`.
|
||||||
|
fn versions(store: &SingleFileMessagePackBackingStore) -> Vec<(String, String, String)> {
|
||||||
|
let catalog = store.pull_schema_catalog().unwrap();
|
||||||
|
let mut rows: Vec<_> = store
|
||||||
|
.pull_all()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|envelope| {
|
||||||
|
let id = envelope.schema_id.as_deref().unwrap();
|
||||||
|
let entry = resolve_schema_entry(&catalog, id).unwrap_or_else(|| panic!("no entry for {id}"));
|
||||||
|
(envelope.key.clone(), entry.schema_name.clone(), entry.schema_version.clone())
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
rows.sort();
|
||||||
|
rows
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy(from: PathBuf) -> (tempfile::TempDir, SingleFileMessagePackBackingStore) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("store.cc");
|
||||||
|
std::fs::copy(from, &path).unwrap();
|
||||||
|
(dir, SingleFileMessagePackBackingStore::new(&path))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_consumer_reads_the_version_of_each_type_a_rust_store_holds() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("store.cc");
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<One>().unwrap();
|
||||||
|
cache.register_entry_type::<Two>().unwrap();
|
||||||
|
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&path), ["tests.sc_one", "tests.sc_two"]).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
cache.put("a", &One { text: "1".into() }).unwrap();
|
||||||
|
cache.put("b", &Two { text: "2".into() }).unwrap();
|
||||||
|
drop(cache);
|
||||||
|
|
||||||
|
let consumer = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
assert_eq!(
|
||||||
|
versions(&consumer),
|
||||||
|
vec![
|
||||||
|
("a".to_string(), "tests.sc_one".to_string(), "tests.sc_one.v1".to_string()),
|
||||||
|
("b".to_string(), "tests.sc_two".to_string(), "tests.sc_two.v1".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_id_only_an_entry_lists_as_compatible_resolves_to_that_entry() {
|
||||||
|
let (_dir, store) = copy(vectors("document-variants-c2a/readability/compatible-id-only-v3.bin"));
|
||||||
|
let catalog = store.pull_schema_catalog().unwrap();
|
||||||
|
assert_eq!(catalog.len(), 1);
|
||||||
|
assert_eq!(catalog[0].schema_id, "vectors.item.next");
|
||||||
|
assert_eq!(catalog[0].compatible_schema_ids, vec!["vectors.item".to_string()]);
|
||||||
|
let entry = resolve_schema_entry(&catalog, "vectors.item").unwrap();
|
||||||
|
assert_eq!(entry.schema_id, "vectors.item.next");
|
||||||
|
assert_eq!(entry.schema_version, "vectors.item.v1");
|
||||||
|
assert_eq!(resolve_schema_entry(&catalog, "vectors.nowhere"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_id_an_entry_owns_resolves_to_its_owner_not_to_an_entry_listing_it() {
|
||||||
|
for vector in ["own-id-over-compatible-v3.bin", "compatible-before-owner-v3.bin"] {
|
||||||
|
let (_dir, store) = copy(vectors("document-variants-c2a/readability").join(vector));
|
||||||
|
let catalog = store.pull_schema_catalog().unwrap();
|
||||||
|
assert_eq!(catalog.len(), 2, "{vector}");
|
||||||
|
let entry = resolve_schema_entry(&catalog, "vectors.item").unwrap();
|
||||||
|
assert_eq!((entry.schema_id.as_str(), entry.schema_version.as_str()), ("vectors.item", "vectors.item.v1"), "{vector}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_legacy_array_and_an_absent_file_publish_no_catalog() {
|
||||||
|
let (_dir, store) = copy(vectors("document-variants-c2a/readability/legacy-envelopes.bin"));
|
||||||
|
assert!(!store.pull_all().unwrap().is_empty());
|
||||||
|
assert_eq!(store.pull_schema_catalog().unwrap(), Vec::<SchemaCatalogEntry>::new());
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let absent = SingleFileMessagePackBackingStore::new(dir.path().join("none.cc"));
|
||||||
|
assert_eq!(absent.pull_schema_catalog().unwrap(), Vec::<SchemaCatalogEntry>::new());
|
||||||
|
assert!(!dir.path().join("none.cc").exists() && !dir.path().join("none.cc.lock").exists(), "reading an absent store creates nothing");
|
||||||
|
}
|
||||||
|
|
||||||
|
// `tests/vectors/key-identity/foreign-store.cc` is written by the C# reference: two types, each published with its version.
|
||||||
|
#[test]
|
||||||
|
fn a_store_the_csharp_reference_wrote_gives_its_versions() {
|
||||||
|
let (_dir, store) = copy(vectors("key-identity/foreign-store.cc"));
|
||||||
|
assert_eq!(
|
||||||
|
versions(&store),
|
||||||
|
vec![
|
||||||
|
("j".to_string(), "tests.ki_alpha".to_string(), "tests.ki_alpha.v1".to_string()),
|
||||||
|
("k".to_string(), "tests.ki_gamma".to_string(), "tests.ki_gamma.v1".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
let catalog = store.pull_schema_catalog().unwrap();
|
||||||
|
assert_eq!(catalog[0].content_hash, "sha256:02f0c8b7ea07c11705c1365fc29136ada08c8076209cacfb8d5526229461ce7a");
|
||||||
|
}
|
||||||
|
|
||||||
|
type Entry = (&'static str, &'static str, &'static str, &'static str, &'static str, Vec<&'static str>, Vec<()>);
|
||||||
|
|
||||||
|
// A hand-built v3 store whose versions are not the `<name>.v1` the writers above publish, so a version read from anywhere but the
|
||||||
|
// version slot cannot pass. `shared` is listed by two entries (the first lister wins); `id.c` is owned by one entry and listed by
|
||||||
|
// an earlier one (the owner wins).
|
||||||
|
#[test]
|
||||||
|
fn distinct_versions_come_through_with_the_decoders_resolution() {
|
||||||
|
let catalog: Vec<Entry> = vec![
|
||||||
|
("id.a", "N.a", "9.9.9", "hash-a", "{}", vec!["id.a", "shared"], vec![]),
|
||||||
|
("id.b", "N.b", "7.0", "hash-b", "{}", vec!["id.b", "shared", "id.c"], vec![]),
|
||||||
|
("id.c", "N.c", "3", "hash-c", "{}", vec!["id.c"], vec![]),
|
||||||
|
];
|
||||||
|
let record = |key: &str, id: &str| (key.to_string(), id.to_string(), "2026-11-13T00:00:00Z".to_string(), b"x".to_vec());
|
||||||
|
let records = vec![record("r1", "shared"), record("r2", "id.c"), record("r3", "id.b"), record("r4", "id.a")];
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("store.cc");
|
||||||
|
std::fs::write(&path, rmp_serde::to_vec(&("cultcache.store.v3", catalog, records)).unwrap()).unwrap();
|
||||||
|
|
||||||
|
let store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
let listed = store.pull_schema_catalog().unwrap();
|
||||||
|
assert_eq!(listed.iter().map(|e| e.schema_version.as_str()).collect::<Vec<_>>(), vec!["9.9.9", "7.0", "3"]);
|
||||||
|
assert_eq!(listed[1].content_hash, "hash-b");
|
||||||
|
assert_eq!(listed[1].compatible_schema_ids, vec!["id.b", "shared", "id.c"]);
|
||||||
|
let got: Vec<(String, String, String)> = {
|
||||||
|
let mut rows: Vec<_> = store
|
||||||
|
.pull_all()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|e| {
|
||||||
|
let entry = resolve_schema_entry(&listed, e.schema_id.as_deref().unwrap()).unwrap();
|
||||||
|
(e.key.clone(), entry.schema_name.clone(), entry.schema_version.clone())
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
rows.sort();
|
||||||
|
rows
|
||||||
|
};
|
||||||
|
let row = |k: &str, n: &str, v: &str| (k.to_string(), n.to_string(), v.to_string());
|
||||||
|
assert_eq!(got, vec![row("r1", "N.a", "9.9.9"), row("r2", "N.c", "3"), row("r3", "N.b", "7.0"), row("r4", "N.a", "9.9.9")]);
|
||||||
|
}
|
||||||
|
|
@ -270,7 +270,7 @@ fn every_write_entry_refuses_a_marked_key_before_any_store_is_touched() {
|
||||||
assert_refused_with(&cache.put_prepared_batch(vec![replacing]).unwrap_err(), "marked", &expected);
|
assert_refused_with(&cache.put_prepared_batch(vec![replacing]).unwrap_err(), "marked", &expected);
|
||||||
assert_refused_with(&cache.delete::<UnionSurface>("marked").unwrap_err(), "marked", &expected);
|
assert_refused_with(&cache.delete::<UnionSurface>("marked").unwrap_err(), "marked", &expected);
|
||||||
assert_refused_with(
|
assert_refused_with(
|
||||||
&cache.update::<UnionSurface, _>("marked", |current| current.unwrap()).unwrap_err(),
|
&cache.update::<UnionSurface, _>("marked", |current| current).unwrap_err(),
|
||||||
"marked",
|
"marked",
|
||||||
&expected,
|
&expected,
|
||||||
);
|
);
|
||||||
|
|
|
||||||
58
packages/cultcache-rs/tests/stored_at_stamp_vector.rs
Normal file
58
packages/cultcache-rs/tests/stored_at_stamp_vector.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
//! `tests/vectors/stored-at-stamp`: the one shape a peer's `storedAt` must parse as. Every row is walked against
|
||||||
|
//! `is_stored_at_stamp`, and each row that is no stamp against the cache's two raw doors, `put_raw_envelope` and
|
||||||
|
//! `put_prepared_batch`: refused, nothing stored or minted, the store still writable after
|
||||||
|
//! (`variants:ruling:raw-put-without-stamp-refused`).
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use cultcache_rs::{CacheBackingStore, CultCache, DatabaseEntry, SingleFileMessagePackBackingStore, is_stored_at_stamp};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "tests.stamp_note", schema = "tests.stamp_note")]
|
||||||
|
struct Note {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rows() -> Vec<(String, bool)> {
|
||||||
|
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/stored-at-stamp/stamps.msgpack");
|
||||||
|
rmp_serde::from_slice(&std::fs::read(path).unwrap()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_stamp_check_agrees_with_every_vector_row() {
|
||||||
|
let rows = rows();
|
||||||
|
assert!(rows.len() > 30, "the vector was not read");
|
||||||
|
for (text, parses) in rows {
|
||||||
|
assert_eq!(is_stored_at_stamp(&text), parses, "{text:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_raw_envelope_that_is_no_stamp_is_refused_and_nothing_is_stored() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("stamps.cc");
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Note>().unwrap();
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
for (text, parses) in rows() {
|
||||||
|
if parses {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let (mut entry, _) = cache.prepare_entry("a", &Note { text: "peer".into() }).unwrap();
|
||||||
|
entry.stored_at = text.clone();
|
||||||
|
let error = cache.put_raw_envelope(entry.clone()).unwrap_err();
|
||||||
|
assert!(format!("{error:#}").contains("stored_at"), "{text:?}: {error:#}");
|
||||||
|
let error = cache.put_prepared_batch(vec![entry]).unwrap_err();
|
||||||
|
assert!(format!("{error:#}").contains("stored_at"), "{text:?}: {error:#}");
|
||||||
|
assert!(cache.get::<Note>("a").unwrap().is_none(), "{text:?} was stored");
|
||||||
|
assert!(!path.exists() || SingleFileMessagePackBackingStore::new(&path).pull_all().unwrap().is_empty(), "{text:?} reached the file");
|
||||||
|
}
|
||||||
|
// Refused writes left the store writable: a stamped one lands under the stamp it carries.
|
||||||
|
let (mut entry, _) = cache.prepare_entry("b", &Note { text: "stamped".into() }).unwrap();
|
||||||
|
entry.stored_at = "2026-11-07T00:00:00Z".into();
|
||||||
|
cache.put_raw_envelope(entry).unwrap();
|
||||||
|
assert_eq!(cache.get::<Note>("b").unwrap().unwrap().text, "stamped");
|
||||||
|
assert!(cache.get::<Note>("a").unwrap().is_none());
|
||||||
|
}
|
||||||
|
|
@ -674,7 +674,7 @@ fn a_refused_load_refuses_every_write() {
|
||||||
("put_prepared_batch", cache.put_prepared_batch(vec![staged.clone()]).unwrap_err()),
|
("put_prepared_batch", cache.put_prepared_batch(vec![staged.clone()]).unwrap_err()),
|
||||||
("put_envelope", cache.put_envelope::<UnionSurface>(staged.clone()).unwrap_err()),
|
("put_envelope", cache.put_envelope::<UnionSurface>(staged.clone()).unwrap_err()),
|
||||||
("put_raw_envelope", cache.put_raw_envelope(staged.clone()).unwrap_err()),
|
("put_raw_envelope", cache.put_raw_envelope(staged.clone()).unwrap_err()),
|
||||||
("update", cache.update::<UnionSurface, _>("good", |current| current.unwrap_or_else(constructed)).unwrap_err()),
|
("update", cache.update::<UnionSurface, _>("good", |current| Some(current.unwrap_or_else(constructed))).unwrap_err()),
|
||||||
("delete", cache.delete::<UnionSurface>("good").unwrap_err()),
|
("delete", cache.delete::<UnionSurface>("good").unwrap_err()),
|
||||||
("delete a key the image lacks", cache.delete::<UnionSurface>("absent").unwrap_err()),
|
("delete a key the image lacks", cache.delete::<UnionSurface>("absent").unwrap_err()),
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -221,7 +221,7 @@ fn a_record_rewriting_its_own_key_keeps_its_value_and_a_change_releases_the_old_
|
||||||
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap().unwrap().sku, "S");
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "S").unwrap().unwrap().sku, "S");
|
||||||
assert_eq!(cache.get_by_index::<Unique>("Sku", "T").unwrap().unwrap().sku, "T");
|
assert_eq!(cache.get_by_index::<Unique>("Sku", "T").unwrap().unwrap().sku, "T");
|
||||||
// Updating through `update` is the same rule.
|
// Updating through `update` is the same rule.
|
||||||
assert!(cache.update::<Unique, _>("k", |_| unique("S")).is_err());
|
assert!(cache.update::<Unique, _>("k", |_| Some(unique("S"))).is_err());
|
||||||
assert_eq!(cache.get::<Unique>("k").unwrap().unwrap().sku, "T");
|
assert_eq!(cache.get::<Unique>("k").unwrap().unwrap().sku, "T");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -326,7 +326,7 @@ fn every_write_path_refuses_and_changes_nothing() {
|
||||||
assert_eq!(text, refusal(first, second, "Multi", alias));
|
assert_eq!(text, refusal(first, second, "Multi", alias));
|
||||||
};
|
};
|
||||||
pick(&error_text(cache.put("m:new", clash)));
|
pick(&error_text(cache.put("m:new", clash)));
|
||||||
pick(&error_text(cache.update::<Multi, _>("m:new", |_| clash.clone())));
|
pick(&error_text(cache.update::<Multi, _>("m:new", |_| Some(clash.clone()))));
|
||||||
let envelope = staged(&cache, &[("m:new", clash.clone())]).remove(0);
|
let envelope = staged(&cache, &[("m:new", clash.clone())]).remove(0);
|
||||||
pick(&error_text(cache.put_envelope::<Multi>(envelope.clone())));
|
pick(&error_text(cache.put_envelope::<Multi>(envelope.clone())));
|
||||||
pick(&error_text(cache.put_raw_envelope(envelope.clone())));
|
pick(&error_text(cache.put_raw_envelope(envelope.clone())));
|
||||||
|
|
|
||||||
770
packages/cultcache-rs/tests/update.rs
Normal file
770
packages/cultcache-rs/tests/update.rs
Normal file
|
|
@ -0,0 +1,770 @@
|
||||||
|
//! `CultCache::update`: read the record as its home store holds it, apply, commit only over what was read, rerun on a lost race
|
||||||
|
//! (C# `CultCache.Update`, as amended by PR #170). A rival is a second `CultCache` on the same file.
|
||||||
|
|
||||||
|
use std::cell::Cell;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::sync::{Arc, Barrier};
|
||||||
|
|
||||||
|
use cultcache_rs::{
|
||||||
|
CacheBackingStore, CultCache, CultCacheEnvelope, CultCommitIf, CultStoredRecord, CultUpdateOutcome, DatabaseEntry,
|
||||||
|
PushAllOptions, RedbMessagePackBackingStore, SingleFileMessagePackBackingStore,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "counter")]
|
||||||
|
struct Counter {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
n: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same type with a second slot: a record it writes carries a slot `Counter` does not read.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "counter")]
|
||||||
|
struct WideCounter {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
n: u32,
|
||||||
|
#[cultcache(key = 1)]
|
||||||
|
extra: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "note")]
|
||||||
|
struct Note {
|
||||||
|
#[cultcache(key = 0)]
|
||||||
|
text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||||
|
#[cultcache(type = "unique_sku")]
|
||||||
|
struct Sku {
|
||||||
|
#[cultcache(key = 0, index = "Sku")]
|
||||||
|
sku: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn counter(n: u32) -> Counter {
|
||||||
|
Counter { n }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open(path: &Path) -> CultCache {
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Counter>().unwrap();
|
||||||
|
cache.register_entry_type::<Note>().unwrap();
|
||||||
|
cache.register_entry_type::<Sku>().unwrap();
|
||||||
|
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path)).unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
cache
|
||||||
|
}
|
||||||
|
|
||||||
|
fn path_in(dir: &tempfile::TempDir) -> PathBuf {
|
||||||
|
dir.path().join("store.cc")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a fresh cache reads at `key`.
|
||||||
|
fn stored(path: &Path, key: &str) -> Option<u32> {
|
||||||
|
open(path).get::<Counter>(key).unwrap().map(|counter| counter.n)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn file(path: &Path) -> Option<Vec<u8>> {
|
||||||
|
std::fs::read(path).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn two_writers_updating_lose_no_increment() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let start = Arc::new(Barrier::new(2));
|
||||||
|
let threads: Vec<_> = (0..2)
|
||||||
|
.map(|_| {
|
||||||
|
let (path, calls, start) = (path.clone(), calls.clone(), start.clone());
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let mut cache = open(&path);
|
||||||
|
start.wait();
|
||||||
|
for _ in 0..50 {
|
||||||
|
let result = cache
|
||||||
|
.update_with_attempts::<Counter, _>("c", 100, |current| {
|
||||||
|
calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
// The window a rival can land in.
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||||
|
Some(counter(current.map_or(0, |held| held.n) + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.outcome, CultUpdateOutcome::Updated);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
for thread in threads {
|
||||||
|
thread.join().unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(stored(&path, "c"), Some(100));
|
||||||
|
assert!(calls.load(Ordering::SeqCst) > 100, "no apply was rerun, so no race was exercised");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_lost_attempt_reruns_apply_on_the_fresh_value() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let mut rival = open(&path);
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
let result = cache
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
let current = current.unwrap().n;
|
||||||
|
if seen.is_empty() {
|
||||||
|
rival.put("c", &counter(100)).unwrap();
|
||||||
|
}
|
||||||
|
seen.push(current);
|
||||||
|
Some(counter(current + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(seen, [1, 100]);
|
||||||
|
assert_eq!(result.outcome, CultUpdateOutcome::Updated);
|
||||||
|
assert_eq!(result.document, Some(counter(101)));
|
||||||
|
assert_eq!(stored(&path, "c"), Some(101));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exhausted_attempts_return_lost_race() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let mut rival = open(&path);
|
||||||
|
let mut calls = 0;
|
||||||
|
let result = cache
|
||||||
|
.update_with_attempts::<Counter, _>("c", 3, |current| {
|
||||||
|
calls += 1;
|
||||||
|
rival.put("c", &counter(100 * calls)).unwrap();
|
||||||
|
current
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(calls, 3);
|
||||||
|
assert_eq!(result.outcome, CultUpdateOutcome::LostRace);
|
||||||
|
// The pull before LostRace shows the value that won.
|
||||||
|
assert_eq!(result.document, Some(counter(300)));
|
||||||
|
assert_eq!(stored(&path, "c"), Some(300));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn max_attempts_bounds() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let mut rival = open(&path);
|
||||||
|
// Always loses; panics past 10 calls so a weakened bound fails here instead of hanging.
|
||||||
|
let mut losing = |calls: &Cell<u32>| {
|
||||||
|
let n = calls.get() + 1;
|
||||||
|
calls.set(n);
|
||||||
|
assert!(n <= 10, "apply ran past the bound");
|
||||||
|
rival.put("c", &counter(1000 + n)).unwrap();
|
||||||
|
Some(counter(n))
|
||||||
|
};
|
||||||
|
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
let one = cache.update_with_attempts::<Counter, _>("c", 1, |_| losing(&calls)).unwrap();
|
||||||
|
assert_eq!((calls.get(), one.outcome), (1, CultUpdateOutcome::LostRace));
|
||||||
|
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
assert!(cache.update_with_attempts::<Counter, _>("c", 0, |_| losing(&calls)).is_err());
|
||||||
|
assert_eq!(calls.get(), 0);
|
||||||
|
|
||||||
|
// The default bound is 5.
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
let default = cache.update::<Counter, _>("c", |_| losing(&calls)).unwrap();
|
||||||
|
assert_eq!((calls.get(), default.outcome), (5, CultUpdateOutcome::LostRace));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn none_declines_and_writes_nothing() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
let absent = cache.update::<Counter, _>("c", |current| {
|
||||||
|
assert_eq!(current, None);
|
||||||
|
None
|
||||||
|
});
|
||||||
|
let absent = absent.unwrap();
|
||||||
|
assert_eq!((absent.outcome, absent.document), (CultUpdateOutcome::Declined, None));
|
||||||
|
assert_eq!(file(&path), None, "a declined update created the store");
|
||||||
|
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let before = file(&path);
|
||||||
|
let held = cache.update::<Counter, _>("c", |_| None).unwrap();
|
||||||
|
assert_eq!((held.outcome, held.document), (CultUpdateOutcome::Declined, Some(counter(1))));
|
||||||
|
assert_eq!(file(&path), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_declined_update_installs_nothing() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let mut rival = open(&path);
|
||||||
|
rival.put("c", &counter(7)).unwrap();
|
||||||
|
let mut seen = None;
|
||||||
|
let result = cache
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
seen = current;
|
||||||
|
None
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(seen, Some(counter(7)), "apply saw the held copy, not the store's");
|
||||||
|
assert_eq!(result.document, Some(counter(1)));
|
||||||
|
assert_eq!(cache.get::<Counter>("c").unwrap(), Some(counter(1)));
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
assert_eq!(cache.get::<Counter>("c").unwrap(), Some(counter(7)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn absent_key_rival_removal_and_a_store_holding_nothing() {
|
||||||
|
// Two writers creating one absent key.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let start = Arc::new(Barrier::new(2));
|
||||||
|
let threads: Vec<_> = (0..2)
|
||||||
|
.map(|_| {
|
||||||
|
let (path, start) = (path.clone(), start.clone());
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let mut cache = open(&path);
|
||||||
|
start.wait();
|
||||||
|
let result = cache
|
||||||
|
.update_with_attempts::<Counter, _>("new", 20, |current| {
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(1));
|
||||||
|
Some(counter(current.map_or(0, |held| held.n) + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.outcome, CultUpdateOutcome::Updated);
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
for thread in threads {
|
||||||
|
thread.join().unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(stored(&path, "new"), Some(2));
|
||||||
|
|
||||||
|
// A rival deletes the record inside apply: the retry sees nothing.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let mut rival = open(&path);
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
let result = cache
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
if seen.is_empty() {
|
||||||
|
assert!(rival.delete::<Counter>("c").unwrap());
|
||||||
|
}
|
||||||
|
seen.push(current.clone());
|
||||||
|
Some(counter(current.map_or(0, |held| held.n) + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(seen, [Some(counter(1)), None]);
|
||||||
|
assert_eq!(result.outcome, CultUpdateOutcome::Updated);
|
||||||
|
assert_eq!(stored(&path, "c"), Some(1));
|
||||||
|
|
||||||
|
// The cache holds a record the store no longer has: apply is handed None.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(5)).unwrap();
|
||||||
|
assert!(open(&path).delete::<Counter>("c").unwrap());
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
cache
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
seen.push(current);
|
||||||
|
Some(counter(1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(seen, [None]);
|
||||||
|
assert_eq!(stored(&path, "c"), Some(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_rival_writing_and_restoring_the_value_still_reruns_apply() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let mut rival = open(&path);
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
cache
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
let current = current.unwrap().n;
|
||||||
|
if seen.is_empty() {
|
||||||
|
rival.put("c", &counter(5)).unwrap();
|
||||||
|
rival.put("c", &counter(1)).unwrap();
|
||||||
|
}
|
||||||
|
seen.push(current);
|
||||||
|
Some(counter(current + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(seen, [1, 1]);
|
||||||
|
assert_eq!(stored(&path, "c"), Some(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_stale_caches_first_attempt_reads_the_store() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut a = open(&path);
|
||||||
|
a.put("c", &counter(1)).unwrap();
|
||||||
|
let mut b = open(&path);
|
||||||
|
a.put("c", &counter(7)).unwrap();
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
b.update::<Counter, _>("c", |current| {
|
||||||
|
seen.push(current.clone());
|
||||||
|
current.map(|held| counter(held.n + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(seen, [Some(counter(7))]);
|
||||||
|
assert_eq!(stored(&path, "c"), Some(8));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A custom store over the single-file store that has no `read` or `commit_if`.
|
||||||
|
struct ReadlessStore(SingleFileMessagePackBackingStore);
|
||||||
|
|
||||||
|
impl CacheBackingStore for ReadlessStore {
|
||||||
|
fn pull_all(&self) -> anyhow::Result<Vec<CultCacheEnvelope>> {
|
||||||
|
self.0.pull_all()
|
||||||
|
}
|
||||||
|
fn push(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<String> {
|
||||||
|
self.0.push(entry)
|
||||||
|
}
|
||||||
|
fn delete(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<()> {
|
||||||
|
self.0.delete(entry)
|
||||||
|
}
|
||||||
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> anyhow::Result<Vec<String>> {
|
||||||
|
self.0.push_all(entries, options)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runs `update` with an apply that counts its calls; the refusal must come before the first.
|
||||||
|
fn refused(cache: &mut CultCache, path: &Path, key: &str) -> String {
|
||||||
|
let before = file(path);
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
let error = cache
|
||||||
|
.update::<Counter, _>(key, |current| {
|
||||||
|
calls.set(calls.get() + 1);
|
||||||
|
current
|
||||||
|
})
|
||||||
|
.expect_err("the update must be refused");
|
||||||
|
assert_eq!(calls.get(), 0, "apply ran before the refusal: {error:#}");
|
||||||
|
assert_eq!(file(path), before, "a refused update touched the store");
|
||||||
|
format!("{error:#}")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn refusals_come_before_apply() {
|
||||||
|
// Another type's record written at the key since the pull: the read resolves it.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
open(&path).put("k", &Note { text: "n".into() }).unwrap();
|
||||||
|
let text = refused(&mut cache, &path, "k");
|
||||||
|
assert!(text.contains("update cannot copy it"), "{text}");
|
||||||
|
|
||||||
|
// Another type held at the key.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("k", &Note { text: "n".into() }).unwrap();
|
||||||
|
refused(&mut cache, &path, "k");
|
||||||
|
|
||||||
|
// A cache with no store holds another type at the key.
|
||||||
|
let mut memory = CultCache::new();
|
||||||
|
memory.register_entry_type::<Counter>().unwrap();
|
||||||
|
memory.register_entry_type::<Note>().unwrap();
|
||||||
|
memory.put("k", &Note { text: "n".into() }).unwrap();
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
assert!(memory
|
||||||
|
.update::<Counter, _>("k", |current| {
|
||||||
|
calls.set(calls.get() + 1);
|
||||||
|
current
|
||||||
|
})
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(calls.get(), 0);
|
||||||
|
|
||||||
|
// A foreign record (no registered type claims it) held at the key, and one that appears after the pull.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
open(&path).put("k", &Note { text: "n".into() }).unwrap();
|
||||||
|
let mut only_counter = CultCache::new();
|
||||||
|
only_counter.register_entry_type::<Counter>().unwrap();
|
||||||
|
only_counter.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
only_counter.pull_all_backing_stores().unwrap();
|
||||||
|
refused(&mut only_counter, &path, "k");
|
||||||
|
let mut later = CultCache::new();
|
||||||
|
later.register_entry_type::<Counter>().unwrap();
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
later.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
later.pull_all_backing_stores().unwrap();
|
||||||
|
open(&path).put("z", &Note { text: "n".into() }).unwrap();
|
||||||
|
let text = refused(&mut later, &path, "z");
|
||||||
|
assert!(text.contains("update cannot copy it"), "{text}");
|
||||||
|
|
||||||
|
// A redb home and a custom store without read.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let redb_path = dir.path().join("store.redb");
|
||||||
|
let mut redb = CultCache::new();
|
||||||
|
redb.register_entry_type::<Counter>().unwrap();
|
||||||
|
redb.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb_path).unwrap()).unwrap();
|
||||||
|
redb.pull_all_backing_stores().unwrap();
|
||||||
|
redb.put("c", &counter(1)).unwrap();
|
||||||
|
let text = refused(&mut redb, &redb_path, "c");
|
||||||
|
assert!(text.contains("read and commit_if"), "{text}");
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut custom = CultCache::new();
|
||||||
|
custom.register_entry_type::<Counter>().unwrap();
|
||||||
|
custom.add_generic_backing_store(ReadlessStore(SingleFileMessagePackBackingStore::new(&path))).unwrap();
|
||||||
|
custom.pull_all_backing_stores().unwrap();
|
||||||
|
custom.put("c", &counter(1)).unwrap();
|
||||||
|
let text = refused(&mut custom, &path, "c");
|
||||||
|
assert!(text.contains("read and commit_if"), "{text}");
|
||||||
|
|
||||||
|
// A held read-only record, and a store record with slots the cache never loaded.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut wide = CultCache::new();
|
||||||
|
wide.register_entry_type::<WideCounter>().unwrap();
|
||||||
|
wide.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
wide.pull_all_backing_stores().unwrap();
|
||||||
|
wide.put("held", &WideCounter { n: 1, extra: 2 }).unwrap();
|
||||||
|
let mut cache = open(&path);
|
||||||
|
let text = refused(&mut cache, &path, "held");
|
||||||
|
assert!(text.contains("held"), "{text}");
|
||||||
|
wide.put("unloaded", &WideCounter { n: 1, extra: 2 }).unwrap();
|
||||||
|
refused(&mut cache, &path, "unloaded");
|
||||||
|
|
||||||
|
// An unpulled cache.
|
||||||
|
let mut unpulled = CultCache::new();
|
||||||
|
unpulled.register_entry_type::<Counter>().unwrap();
|
||||||
|
unpulled.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||||
|
refused(&mut unpulled, &path, "held");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_refusal_after_apply_is_not_retried() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("a", &Sku { sku: "S".into() }).unwrap();
|
||||||
|
cache.put("b", &Sku { sku: "T".into() }).unwrap();
|
||||||
|
let before = file(&path);
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
let error = cache
|
||||||
|
.update::<Sku, _>("b", |_| {
|
||||||
|
calls.set(calls.get() + 1);
|
||||||
|
Some(Sku { sku: "S".into() })
|
||||||
|
})
|
||||||
|
.expect_err("a unique-index clash");
|
||||||
|
assert_eq!(calls.get(), 1, "{error:#}");
|
||||||
|
assert_eq!(file(&path), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn own_writes_and_the_store_less_cache() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
cache.put("c", &counter(1)).unwrap();
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
for expected in [2, 3] {
|
||||||
|
let result = cache
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
calls.set(calls.get() + 1);
|
||||||
|
current.map(|held| counter(held.n + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.outcome, CultUpdateOutcome::Updated);
|
||||||
|
assert_eq!(result.document, Some(counter(expected)));
|
||||||
|
assert_eq!(result.document, cache.get::<Counter>("c").unwrap());
|
||||||
|
}
|
||||||
|
assert_eq!(calls.get(), 2);
|
||||||
|
|
||||||
|
let mut memory = CultCache::new();
|
||||||
|
memory.register_entry_type::<Counter>().unwrap();
|
||||||
|
let calls = Cell::new(0);
|
||||||
|
let result = memory
|
||||||
|
.update::<Counter, _>("c", |current| {
|
||||||
|
calls.set(calls.get() + 1);
|
||||||
|
Some(counter(current.map_or(0, |held| held.n) + 1))
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!((calls.get(), result.outcome, result.document), (1, CultUpdateOutcome::Updated, Some(counter(1))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_ledger_holds_under_contention() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let start = Arc::new(Barrier::new(3));
|
||||||
|
let threads: Vec<_> = (0..3)
|
||||||
|
.map(|_| {
|
||||||
|
let (path, start) = (path.clone(), start.clone());
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let mut cache = open(&path);
|
||||||
|
start.wait();
|
||||||
|
(0..25)
|
||||||
|
.filter(|_| {
|
||||||
|
let result = cache
|
||||||
|
.update::<Counter, _>("c", |current| Some(counter(current.map_or(0, |held| held.n) + 1)))
|
||||||
|
.unwrap();
|
||||||
|
result.outcome == CultUpdateOutcome::Updated
|
||||||
|
})
|
||||||
|
.count() as u32
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let updated: u32 = threads.into_iter().map(|thread| thread.join().unwrap()).sum();
|
||||||
|
assert_eq!(stored(&path, "c"), Some(updated));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn envelope_of<T: DatabaseEntry>(cache: &CultCache, key: &str, value: &T, stamp: &str) -> CultCacheEnvelope {
|
||||||
|
let (mut entry, _) = cache.prepare_entry(key, value).unwrap();
|
||||||
|
entry.stored_at = stamp.to_string();
|
||||||
|
entry
|
||||||
|
}
|
||||||
|
|
||||||
|
const FUTURE: &str = "2099-01-01T00:00:00Z";
|
||||||
|
|
||||||
|
fn later_than_future(stamp: &str) -> bool {
|
||||||
|
chrono::DateTime::parse_from_rfc3339(stamp).unwrap() > chrono::DateTime::parse_from_rfc3339(FUTURE).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn commit_if_writes_only_over_the_version_it_was_given() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let cache = open(&path);
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
assert_eq!(store.read("c").unwrap(), None);
|
||||||
|
let one = envelope_of(&cache, "c", &counter(1), FUTURE);
|
||||||
|
// Against an absent key, expecting a version mismatches; expecting nothing commits.
|
||||||
|
assert_eq!(store.commit_if(&one, Some("0000")).unwrap(), CultCommitIf::Mismatch);
|
||||||
|
assert_eq!(file(&path), None);
|
||||||
|
assert_eq!(store.commit_if(&one, None).unwrap(), CultCommitIf::Committed { stored_at: FUTURE.to_string() });
|
||||||
|
let CultStoredRecord { envelope, version } = store.read("c").unwrap().unwrap();
|
||||||
|
assert_eq!(envelope.stored_at, FUTURE);
|
||||||
|
|
||||||
|
let before = file(&path);
|
||||||
|
let two = envelope_of(&cache, "c", &counter(2), FUTURE);
|
||||||
|
assert_eq!(store.commit_if(&two, None).unwrap(), CultCommitIf::Mismatch);
|
||||||
|
assert_eq!(store.commit_if(&two, Some("not-the-version")).unwrap(), CultCommitIf::Mismatch);
|
||||||
|
assert_eq!(file(&path), before, "a mismatch wrote");
|
||||||
|
let CultCommitIf::Committed { stored_at } = store.commit_if(&two, Some(&version)).unwrap() else { panic!("the version matches") };
|
||||||
|
assert!(later_than_future(&stored_at), "{stored_at}");
|
||||||
|
assert_eq!(stored(&path, "c"), Some(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_cache_facing_writes_stamp_over_a_replaced_record() {
|
||||||
|
let seeded = |dir: &tempfile::TempDir| {
|
||||||
|
let path = path_in(dir);
|
||||||
|
let seed = open(&path);
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
store.push(&envelope_of(&seed, "c", &counter(1), FUTURE)).unwrap();
|
||||||
|
(path, seed)
|
||||||
|
};
|
||||||
|
let on_disk = |path: &Path| {
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(path);
|
||||||
|
let stamp = store.read("c").unwrap().unwrap().envelope.stored_at;
|
||||||
|
let _ = &mut store;
|
||||||
|
stamp
|
||||||
|
};
|
||||||
|
|
||||||
|
// The store: push, push_all and commit_if each return the stamp they wrote, strictly later than what they replaced.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (path, seed) = seeded(&dir);
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
let stamp = store.push(&envelope_of(&seed, "c", &counter(2), FUTURE)).unwrap();
|
||||||
|
assert!(later_than_future(&stamp) && stamp == on_disk(&path), "push: {stamp}");
|
||||||
|
let stamps = store.push_all(&[envelope_of(&seed, "c", &counter(3), FUTURE)], PushAllOptions::default()).unwrap();
|
||||||
|
assert!(later_than_future(&stamps[0]) && stamps[0] == on_disk(&path), "push_all: {stamps:?}");
|
||||||
|
let version = store.read("c").unwrap().unwrap().version;
|
||||||
|
let CultCommitIf::Committed { stored_at } = store.commit_if(&envelope_of(&seed, "c", &counter(4), FUTURE), Some(&version)).unwrap()
|
||||||
|
else {
|
||||||
|
panic!("matching version")
|
||||||
|
};
|
||||||
|
assert!(later_than_future(&stored_at) && stored_at == on_disk(&path), "commit_if: {stored_at}");
|
||||||
|
// One stamp per input, in order, a duplicate getting the stamp of the entry that landed; a soft write that wrote nothing none.
|
||||||
|
let stamps = store
|
||||||
|
.push_all(
|
||||||
|
&[envelope_of(&seed, "c", &counter(5), FUTURE), envelope_of(&seed, "d", &counter(6), FUTURE), envelope_of(&seed, "c", &counter(7), FUTURE)],
|
||||||
|
PushAllOptions::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(stamps.len(), 3);
|
||||||
|
assert_eq!(stamps[0], stamps[2]);
|
||||||
|
assert_eq!(stamps[1], FUTURE, "a record replacing nothing keeps its stamp");
|
||||||
|
assert_eq!(stamps[0], on_disk(&path));
|
||||||
|
assert!(store.push_all(&[envelope_of(&seed, "e", &counter(1), FUTURE)], PushAllOptions { soft: true }).unwrap().is_empty());
|
||||||
|
|
||||||
|
// The cache holds the stamp the store wrote after every write entry.
|
||||||
|
type Write = fn(&mut CultCache) -> ();
|
||||||
|
let writes: [(&str, Write); 5] = [
|
||||||
|
("put", |cache| drop(cache.put("c", &counter(9)).unwrap())),
|
||||||
|
("put_envelope", |cache| {
|
||||||
|
let entry = envelope_of(cache, "c", &counter(9), FUTURE);
|
||||||
|
drop(cache.put_envelope::<Counter>(entry).unwrap())
|
||||||
|
}),
|
||||||
|
("put_raw_envelope", |cache| {
|
||||||
|
let entry = envelope_of(cache, "c", &counter(9), FUTURE);
|
||||||
|
cache.put_raw_envelope(entry).unwrap()
|
||||||
|
}),
|
||||||
|
("put_prepared_batch", |cache| {
|
||||||
|
let entry = envelope_of(cache, "c", &counter(9), FUTURE);
|
||||||
|
cache.put_prepared_batch(vec![entry]).unwrap()
|
||||||
|
}),
|
||||||
|
("update", |cache| drop(cache.update::<Counter, _>("c", |_| Some(counter(9))).unwrap())),
|
||||||
|
];
|
||||||
|
for (name, write) in writes {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (path, _) = seeded(&dir);
|
||||||
|
let mut cache = open(&path);
|
||||||
|
write(&mut cache);
|
||||||
|
let held = cache.get_envelope::<Counter>("c").unwrap().unwrap().stored_at;
|
||||||
|
assert!(later_than_future(&held), "{name}: {held}");
|
||||||
|
assert_eq!(held, on_disk(&path), "{name}: the cache holds a stamp the store did not write");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_byte_exact_writers_keep_the_callers_stamp() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let cache = open(&path);
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
let held = envelope_of(&cache, "c", &counter(1), FUTURE);
|
||||||
|
store.push(&held).unwrap();
|
||||||
|
let swapped = envelope_of(&cache, "c", &counter(2), "2026-01-01T00:00:00Z");
|
||||||
|
assert!(store.compare_and_swap_entry(&held, swapped.clone()).unwrap());
|
||||||
|
assert_eq!(store.read("c").unwrap().unwrap().envelope, swapped, "compare_and_swap_entry restamped");
|
||||||
|
|
||||||
|
let redb_path = dir.path().join("store.redb");
|
||||||
|
let mut redb = RedbMessagePackBackingStore::new(&redb_path).unwrap();
|
||||||
|
redb.push(&held).unwrap();
|
||||||
|
let older = envelope_of(&cache, "c", &counter(3), "2026-01-01T00:00:00Z");
|
||||||
|
assert_eq!(redb.push(&older).unwrap(), older.stored_at);
|
||||||
|
assert_eq!(redb.pull_all().unwrap(), vec![older.clone()]);
|
||||||
|
assert_eq!(redb.push_all(std::slice::from_ref(&held), PushAllOptions::default()).unwrap(), vec![held.stored_at.clone()]);
|
||||||
|
assert_eq!(redb.pull_all().unwrap(), vec![held]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A single-file store that reports whatever stamp it is told to, as a custom store might.
|
||||||
|
struct ReportingStore {
|
||||||
|
inner: SingleFileMessagePackBackingStore,
|
||||||
|
report: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CacheBackingStore for ReportingStore {
|
||||||
|
fn pull_all(&self) -> anyhow::Result<Vec<CultCacheEnvelope>> {
|
||||||
|
self.inner.pull_all()
|
||||||
|
}
|
||||||
|
fn push(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<String> {
|
||||||
|
self.inner.push(entry)?;
|
||||||
|
Ok(self.report.clone())
|
||||||
|
}
|
||||||
|
fn delete(&mut self, entry: &CultCacheEnvelope) -> anyhow::Result<()> {
|
||||||
|
self.inner.delete(entry)
|
||||||
|
}
|
||||||
|
fn push_all(&mut self, entries: &[CultCacheEnvelope], options: PushAllOptions) -> anyhow::Result<Vec<String>> {
|
||||||
|
self.inner.push_all(entries, options)?;
|
||||||
|
Ok(entries.iter().map(|_| self.report.clone()).collect())
|
||||||
|
}
|
||||||
|
fn read(&self, key: &str) -> anyhow::Result<Option<CultStoredRecord>> {
|
||||||
|
self.inner.read(key)
|
||||||
|
}
|
||||||
|
fn commit_if(&mut self, entry: &CultCacheEnvelope, expected: Option<&str>) -> anyhow::Result<CultCommitIf> {
|
||||||
|
Ok(match self.inner.commit_if(entry, expected)? {
|
||||||
|
CultCommitIf::Committed { .. } => CultCommitIf::Committed { stored_at: self.report.clone() },
|
||||||
|
CultCommitIf::Mismatch => CultCommitIf::Mismatch,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_cache_holds_a_reported_stamp_only_when_it_parses_as_rfc3339() {
|
||||||
|
type Write = fn(&mut CultCache);
|
||||||
|
let writes: [(&str, Write); 4] = [
|
||||||
|
("put", |cache| drop(cache.put("c", &counter(1)).unwrap())),
|
||||||
|
("put_raw_envelope", |cache| {
|
||||||
|
let entry = envelope_of(cache, "c", &counter(1), "2020-01-01T00:00:00Z");
|
||||||
|
cache.put_raw_envelope(entry).unwrap()
|
||||||
|
}),
|
||||||
|
("put_prepared_batch", |cache| {
|
||||||
|
let entry = envelope_of(cache, "c", &counter(1), "2020-01-01T00:00:00Z");
|
||||||
|
cache.put_prepared_batch(vec![entry]).unwrap()
|
||||||
|
}),
|
||||||
|
("update", |cache| drop(cache.update::<Counter, _>("c", |_| Some(counter(1))).unwrap())),
|
||||||
|
];
|
||||||
|
for (report, parses) in [
|
||||||
|
("2030-01-01T00:00:00Z", true),
|
||||||
|
("2030-01-01T00:00:00.123456+00:00", true),
|
||||||
|
("not a stamp", false),
|
||||||
|
("", false),
|
||||||
|
("2030-01-01", false),
|
||||||
|
("2030-01-01 00:00:00", false),
|
||||||
|
("2030-01-01T00:00:00", false),
|
||||||
|
("1893456000", false),
|
||||||
|
] {
|
||||||
|
for (name, write) in writes {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let mut cache = CultCache::new();
|
||||||
|
cache.register_entry_type::<Counter>().unwrap();
|
||||||
|
cache
|
||||||
|
.add_generic_backing_store(ReportingStore { inner: SingleFileMessagePackBackingStore::new(&path), report: report.to_string() })
|
||||||
|
.unwrap();
|
||||||
|
cache.pull_all_backing_stores().unwrap();
|
||||||
|
write(&mut cache);
|
||||||
|
let held = cache.get_envelope::<Counter>("c").unwrap().unwrap().stored_at;
|
||||||
|
if parses {
|
||||||
|
assert_eq!(held, report, "{name}");
|
||||||
|
} else {
|
||||||
|
assert_ne!(held, report, "{name} held a stamp that is not RFC 3339");
|
||||||
|
assert!(chrono::DateTime::parse_from_rfc3339(&held).is_ok(), "{name}: {held}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_stamp_with_no_successor_refuses_every_single_file_write_and_writes_nothing() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let cache = open(&path);
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
let last = "9999-12-31T23:59:59.999999Z";
|
||||||
|
store.push(&envelope_of(&cache, "c", &counter(1), last)).unwrap();
|
||||||
|
let before = file(&path);
|
||||||
|
let version = store.read("c").unwrap().unwrap().version;
|
||||||
|
let next = envelope_of(&cache, "c", &counter(2), "2020-01-01T00:00:00Z");
|
||||||
|
assert!(store.push(&next).is_err());
|
||||||
|
assert!(store.push_all(std::slice::from_ref(&next), PushAllOptions::default()).is_err());
|
||||||
|
assert!(store.commit_if(&next, Some(&version)).is_err());
|
||||||
|
assert_eq!(file(&path), before, "a refused mint wrote");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn commit_if_refuses_a_retype_as_push_does() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = path_in(&dir);
|
||||||
|
let cache = open(&path);
|
||||||
|
let mut store = SingleFileMessagePackBackingStore::new(&path);
|
||||||
|
let note = envelope_of(&cache, "k", &Note { text: "n".into() }, "2020-01-01T00:00:00Z");
|
||||||
|
store.push(¬e).unwrap();
|
||||||
|
let version = store.read("k").unwrap().unwrap().version;
|
||||||
|
let before = file(&path);
|
||||||
|
let counter_at_k = envelope_of(&cache, "k", &counter(1), "2020-01-01T00:00:00Z");
|
||||||
|
let error = store.commit_if(&counter_at_k, Some(&version)).expect_err("a retype");
|
||||||
|
assert!(error.downcast_ref::<cultcache_rs::SchemaConflictError>().is_some(), "{error:#}");
|
||||||
|
assert!(store.push(&counter_at_k).is_err());
|
||||||
|
assert_eq!(file(&path), before, "a refused retype wrote");
|
||||||
|
assert_eq!(store.read("k").unwrap().unwrap().envelope.r#type, "note");
|
||||||
|
}
|
||||||
|
|
@ -5,12 +5,23 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following
|
||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.16.0] - 2026-11-14
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- `isStoredAtStamp(text)` (package and browser entries): whether a `storedAt` is a stamp, the one shape pinned in `tests/vectors/stored-at-stamp/` (an RFC 3339 date-time with `T`, an optional 1 to 9 digit fraction, `Z` or `+hh:mm`/`-hh:mm`, a real calendar date, seconds 00-59, years 0001-9999). cultnet-ts and cultmesh-ts judge a peer's put by it (variants:ruling:raw-put-without-stamp-refused).
|
||||||
|
- `CacheBackingStore` gains the optional `read(key)` (the record at the key with its content version, from one read) and `commitIf(entry, expectedVersion)` (write only if the version still holds, under the store's write lock; returns the storedAt written). The single-file store implements both. The content version is the lowercase hex SHA-256 of the record's persisted encoding `[key, schemaId, storedAt, payload]`, equal to the C# reference's on `tests/vectors/content-version`. A store without them still works for everything except `update`.
|
||||||
|
- `CultCacheUpdateResult` and `CultCacheUpdateOptions` (`maxAttempts`, default 5), exported from the package entry and the browser entry.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
- A write that replaces a record at its key with different content (another schema id or other payload bytes) is stored strictly later than it: a supplied `storedAt` not later than the stored one is written one millisecond after it, so a content version that includes `storedAt` never repeats across writes. A rewrite of identical content (same schema id and payload bytes) keeps the stored stamp and so its version, which is what keeps a periodic flush of an unchanged cache from changing every record's version; C# re-mints every replacing write, so this is a departure from it, and the writer's type and catalog entry are written either way. `push`, `pushAll` and `commitIf` alike; the single-file store's `push` returns the stamp it wrote (`CacheBackingStore.push` may resolve `{ storedAt }`), and `put`, `putGlobal` and `putEnvelope` install that stamp in the cache, so the cache and the file agree after a put over a record stamped at or after now (a push result that is not a parseable stamp is ignored). A stamp that would pass 9999-12-31T23:59:59.999Z is refused with `RangeError` and nothing is written.
|
||||||
|
- `withStoreWriteLock` looks for a link at the store path again once the lock is held, so a link put at the path while a writer waited on the lock is refused with `StoreLinkedError` and the lock released, instead of being replaced. Pinned by a test that puts the link while another holder owns the lock.
|
||||||
- `CultCache.isWritable` does not look at store paths: a link at a store path is not predicted, and the write refuses it with `StoreLinkedError`. Pinned by a test through a cache attached to a linked store.
|
- `CultCache.isWritable` does not look at store paths: a link at a store path is not predicted, and the write refuses it with `StoreLinkedError`. Pinned by a test through a cache attached to a linked store.
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
|
- `update` and `updateGlobal` are the conditional golden path and no longer the blind pair. `update(definition, key, apply, options?)` reads the record and its content version from the store, passes `apply` a fresh copy (never the held instance), commits only if the version still holds and runs `apply` again on a lost race, up to `maxAttempts`. `apply` may be async, may return `undefined` to write nothing, and must be free of side effects. It resolves to `{ outcome: "updated" | "declined" | "lostRace", document }` instead of the written value, so a caller reads `result.document`; a lost race that outlasts the attempts is `lostRace`, not a throw. `apply` returning `undefined` or `null` declines. `maxAttempts` is an integer of at least 1; `undefined` is the default 5 and `null` is refused with `RangeError`. A conditional write over a global loaded from a legacy key is refused wherever it enters (`updateGlobal`, or `update` at `CultCache.GLOBAL_KEY`) and leaves the legacy record untouched; `apply` may have run once before the write is refused. A store attached to a store-less cache while `apply` awaits makes that attempt a lost race. It needs a store with `read` and `commitIf` (the single-file store has them); a cache with no store compares the held entry. `put` and `putGlobal` are the blind writes and are unchanged.
|
||||||
- A write at a single-file store path that is a symbolic link, live or dangling, is refused with the new `StoreLinkedError` (exported from the package entry; `error.path` names the path) before the lock and the replace, and writes nothing: `push`, `pushAll`, `delete` and a soft flush alike. Before, a write at a live link replaced the link with a regular file, a dangling link failed with `ENOENT` (or, for `pushAll`, was replaced), and a soft flush at a link skipped. A caller catching `ENOENT` around a write at a dangling link now catches `StoreLinkedError`; configure the link's target as the store path. Reads through a live link work. An `lstat` failure other than `ENOENT` (`ENOTDIR`, `EACCES`, `ELOOP`) is rethrown as it is.
|
- A write at a single-file store path that is a symbolic link, live or dangling, is refused with the new `StoreLinkedError` (exported from the package entry; `error.path` names the path) before the lock and the replace, and writes nothing: `push`, `pushAll`, `delete` and a soft flush alike. Before, a write at a live link replaced the link with a regular file, a dangling link failed with `ENOENT` (or, for `pushAll`, was replaced), and a soft flush at a link skipped. A caller catching `ENOENT` around a write at a dangling link now catches `StoreLinkedError`; configure the link's target as the store path. Reads through a live link work. An `lstat` failure other than `ENOENT` (`ENOTDIR`, `EACCES`, `ELOOP`) is rethrown as it is.
|
||||||
- `SingleFileMessagePackBackingStore.pushAll` writes the given entries onto the store as it is under the lock, as `push` does: a record under the same key is replaced and no other is removed. It used to replace the whole file with the entries, so `CultMesh.flush()` from a cache that held less than the store (a node created with `pullOnStart: false`, or one whose image was stale) wiped the records it never read. A caller that relied on `pushAll` to reset a store removes the file first. A soft flush is unchanged. `CultCache.storedSnapshot()` is the image as the stores hold it (a global loaded from a legacy key under that key) and is what `CultMesh.flush` persists, so a flush no longer leaves such a global under two keys; `snapshot()` is unchanged.
|
- `SingleFileMessagePackBackingStore.pushAll` writes the given entries onto the store as it is under the lock, as `push` does: a record under the same key is replaced and no other is removed. It used to replace the whole file with the entries, so `CultMesh.flush()` from a cache that held less than the store (a node created with `pullOnStart: false`, or one whose image was stale) wiped the records it never read. A caller that relied on `pushAll` to reset a store removes the file first. A soft flush is unchanged. `CultCache.storedSnapshot()` is the image as the stores hold it (a global loaded from a legacy key under that key) and is what `CultMesh.flush` persists, so a flush no longer leaves such a global under two keys; `snapshot()` is unchanged.
|
||||||
- A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
- A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
||||||
|
|
@ -21,7 +32,7 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following
|
||||||
the many-row selection aliases (never indexed for lookup, same catalog `indexAlias`). The per-key migration rule is
|
the many-row selection aliases (never indexed for lookup, same catalog `indexAlias`). The per-key migration rule is
|
||||||
in the README; classify every key before the CultLib your tools load is updated. The refusal verdict is the contract, its text is informational.
|
in the README; classify every key before the CultLib your tools load is updated. The refusal verdict is the contract, its text is informational.
|
||||||
- The single-file reader (v1 and the legacy envelope array) refuses a store that holds two records under one key as `StoreUnreadableError`; a store that opened before may now be refused. Repair it by removing the extra record with the runtime that wrote it.
|
- The single-file reader (v1 and the legacy envelope array) refuses a store that holds two records under one key as `StoreUnreadableError`; a store that opened before may now be refused. Repair it by removing the extra record with the runtime that wrote it.
|
||||||
- A `put`, `push` or `pushAll` that would land a second type under a held key (against the file, or within one batch) is refused with `SchemaConflictError`, and nothing is written. The written type may replace the record when its catalog entry carries the stored schema id as its own or lists it compatible (a type renamed under a stable id restamps the record, as C# does); otherwise retyping a key is a delete, then a put. The C# allowance for an undeclared older id of the same schema name that reads whole is not ported: a same-named type keeps passing, including one that drops persisted slots, which C# refuses.
|
- A `put`, `push` or `pushAll` that would land a second type under a held key (against the file, or within one batch) is refused with `SchemaConflictError`, and nothing is written. The written type may replace the record when its catalog entry carries the stored schema id as its own or lists it compatible (a type renamed under a stable id restamps the record, as C# does, when its payload changes; an identical payload keeps the stamp); otherwise retyping a key is a delete, then a put. The C# allowance for an undeclared older id of the same schema name that reads whole is not ported: a same-named type keeps passing, including one that drops persisted slots, which C# refuses.
|
||||||
- Same-type writes replace by key; in one `pushAll` batch the last entry for a key wins.
|
- Same-type writes replace by key; in one `pushAll` batch the last entry for a key wins.
|
||||||
- A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`.
|
- A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`.
|
||||||
- The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string.
|
- The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string.
|
||||||
|
|
|
||||||
|
|
@ -168,10 +168,8 @@ const settings = cache.getRequiredGlobal(settingsDocument);
|
||||||
- `getByIndex(...)`
|
- `getByIndex(...)`
|
||||||
- `getGlobal(...)`
|
- `getGlobal(...)`
|
||||||
- `getRequiredGlobal(...)`
|
- `getRequiredGlobal(...)`
|
||||||
- `put(...)`
|
- `update(...)` and `updateGlobal(...)`, the conditional golden path: apply a function to a fresh copy of the record as the store holds it, commit only if it is unchanged, re-run on a lost race (`updated`, `declined` or `lostRace`)
|
||||||
- `putGlobal(...)`
|
- `put(...)` and `putGlobal(...)`, the blind opt-out: last writer wins
|
||||||
- `update(...)`
|
|
||||||
- `updateGlobal(...)`
|
|
||||||
- `delete(...)`
|
- `delete(...)`
|
||||||
- `deleteGlobal(...)`
|
- `deleteGlobal(...)`
|
||||||
- `snapshot()` (the logical image) and `storedSnapshot()` (the image as the stores hold it, for a whole flush)
|
- `snapshot()` (the logical image) and `storedSnapshot()` (the image as the stores hold it, for a whole flush)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@gamecult/cultcache-ts",
|
"name": "@gamecult/cultcache-ts",
|
||||||
"version": "0.15.0",
|
"version": "0.16.0",
|
||||||
"description": "CultCache-style typed persistence for TypeScript with schema-backed polymorphic MessagePack storage.",
|
"description": "CultCache-style typed persistence for TypeScript with schema-backed polymorphic MessagePack storage.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"types": "dist/index.d.ts",
|
"types": "dist/index.d.ts",
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ export {
|
||||||
} from "./document";
|
} from "./document";
|
||||||
export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth";
|
export { CultCachePayloadDepthError, MAX_PAYLOAD_DEPTH } from "./payload-depth";
|
||||||
export type { DocumentPayload } from "./document";
|
export type { DocumentPayload } from "./document";
|
||||||
|
export { isStoredAtStamp } from "./stored-at";
|
||||||
export { SchemaConflictError, StoreUnreadableError } from "./store-format";
|
export { SchemaConflictError, StoreUnreadableError } from "./store-format";
|
||||||
export type {
|
export type {
|
||||||
AnyCultCacheDocumentDefinition,
|
AnyCultCacheDocumentDefinition,
|
||||||
|
|
@ -35,6 +36,8 @@ export type {
|
||||||
InferCultCacheSchemaValue,
|
InferCultCacheSchemaValue,
|
||||||
CultCacheStoreRegistration,
|
CultCacheStoreRegistration,
|
||||||
CultCacheUnreadSlots,
|
CultCacheUnreadSlots,
|
||||||
|
CultCacheUpdateOptions,
|
||||||
|
CultCacheUpdateResult,
|
||||||
PushAllOptions,
|
PushAllOptions,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
export type {
|
export type {
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,8 @@ import type {
|
||||||
CultCacheSchemaCatalogEntry,
|
CultCacheSchemaCatalogEntry,
|
||||||
CultCacheStoreRegistration,
|
CultCacheStoreRegistration,
|
||||||
CultCacheUnreadSlots,
|
CultCacheUnreadSlots,
|
||||||
|
CultCacheUpdateOptions,
|
||||||
|
CultCacheUpdateResult,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
type Accessor = (value: unknown) => string | undefined;
|
type Accessor = (value: unknown) => string | undefined;
|
||||||
|
|
@ -48,6 +50,10 @@ type EntryLookups = {
|
||||||
readonly indexes: ReadonlyMap<string, string>;
|
readonly indexes: ReadonlyMap<string, string>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// What a conditional write requires of the record at its key: the content version its store reported (null: no record), or for a
|
||||||
|
// cache with no store the entry it held. Every write installs a new entry object, so the identity catches an A-B-A.
|
||||||
|
type WriteCondition = { readonly version: string | null } | { readonly held: HydratedEntry | undefined };
|
||||||
|
|
||||||
type HydratedEntry = CultCacheEnvelope & {
|
type HydratedEntry = CultCacheEnvelope & {
|
||||||
catalogEntry: NonNullable<CultCacheEnvelope["catalogEntry"]>;
|
catalogEntry: NonNullable<CultCacheEnvelope["catalogEntry"]>;
|
||||||
value: unknown;
|
value: unknown;
|
||||||
|
|
@ -302,85 +308,88 @@ export class CultCache {
|
||||||
// HydratedState, running every check and every user accessor, and replaces the cache's state
|
// HydratedState, running every check and every user accessor, and replaces the cache's state
|
||||||
// in one assignment only once the build finishes. A refused load admits nothing.
|
// in one assignment only once the build finishes. A refused load admits nothing.
|
||||||
pullAllBackingStores(): Promise<void> {
|
pullAllBackingStores(): Promise<void> {
|
||||||
return this.#serial(async () => {
|
return this.#serial(() => this.#pullNow());
|
||||||
const next = emptyHydratedState();
|
}
|
||||||
const persistedGlobalKeys = new Map<string, string>();
|
|
||||||
|
|
||||||
for (const registration of this.#stores) {
|
// The pull itself, for a caller already inside a serial operation.
|
||||||
const entries = await registration.store.pullAll();
|
async #pullNow(): Promise<void> {
|
||||||
|
const next = emptyHydratedState();
|
||||||
|
const persistedGlobalKeys = new Map<string, string>();
|
||||||
|
|
||||||
for (const entry of entries) {
|
for (const registration of this.#stores) {
|
||||||
const registered = this.#resolveDefinitionForEnvelope(entry);
|
const entries = await registration.store.pullAll();
|
||||||
if (!registered) {
|
|
||||||
// A store may hold what this cache has no type for; it is carried, not served, and not a reason to refuse the load.
|
|
||||||
next.foreign.push({
|
|
||||||
key: entry.key,
|
|
||||||
schemaId: schemaIdForEnvelope(entry),
|
|
||||||
schemaName: entry.catalogEntry?.schemaName ?? entry.type,
|
|
||||||
storedAt: entry.storedAt,
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const type = registered.definition.type;
|
for (const entry of entries) {
|
||||||
const home = this.#resolveRegistration(type);
|
const registered = this.#resolveDefinitionForEnvelope(entry);
|
||||||
if (home?.store !== registration.store) {
|
if (!registered) {
|
||||||
|
// A store may hold what this cache has no type for; it is carried, not served, and not a reason to refuse the load.
|
||||||
|
next.foreign.push({
|
||||||
|
key: entry.key,
|
||||||
|
schemaId: schemaIdForEnvelope(entry),
|
||||||
|
schemaName: entry.catalogEntry?.schemaName ?? entry.type,
|
||||||
|
storedAt: entry.storedAt,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const type = registered.definition.type;
|
||||||
|
const home = this.#resolveRegistration(type);
|
||||||
|
if (home?.store !== registration.store) {
|
||||||
|
throw new Error(
|
||||||
|
`CultCache "${type}" record "${entry.key}" was loaded from ${this.#describe(registration)}, ` +
|
||||||
|
`but its home is ${this.#describe(home)}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = this.#cloneBytes(entry.payload);
|
||||||
|
const { value, unreadSlots } = registered.formatter.decode(payload);
|
||||||
|
let admitted: CultCacheEnvelope = { ...entry, type, payload };
|
||||||
|
if (registered.global) {
|
||||||
|
const persistedKey = persistedGlobalKeys.get(type);
|
||||||
|
if (persistedKey !== undefined) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`CultCache "${type}" record "${entry.key}" was loaded from ${this.#describe(registration)}, ` +
|
`CultCache global document type "${type}" has multiple persisted entries: "${persistedKey}" and "${entry.key}".`,
|
||||||
`but its home is ${this.#describe(home)}.`,
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
persistedGlobalKeys.set(type, entry.key);
|
||||||
const payload = this.#cloneBytes(entry.payload);
|
if (entry.key !== CultCache.GLOBAL_KEY) {
|
||||||
const { value, unreadSlots } = registered.formatter.decode(payload);
|
next.legacyGlobals.set(type, admitted);
|
||||||
let admitted: CultCacheEnvelope = { ...entry, type, payload };
|
admitted = { ...admitted, key: CultCache.GLOBAL_KEY };
|
||||||
if (registered.global) {
|
|
||||||
const persistedKey = persistedGlobalKeys.get(type);
|
|
||||||
if (persistedKey !== undefined) {
|
|
||||||
throw new Error(
|
|
||||||
`CultCache global document type "${type}" has multiple persisted entries: "${persistedKey}" and "${entry.key}".`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
persistedGlobalKeys.set(type, entry.key);
|
|
||||||
if (entry.key !== CultCache.GLOBAL_KEY) {
|
|
||||||
next.legacyGlobals.set(type, admitted);
|
|
||||||
admitted = { ...admitted, key: CultCache.GLOBAL_KEY };
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
const hydrated = this.#admitEntry(next, registered, admitted, value, unreadSlots);
|
|
||||||
const other = this.#heldUnderOtherType(next, type, hydrated.key);
|
|
||||||
if (other !== undefined) {
|
|
||||||
throw new Error(`Record ${hydrated.key} is held by two stores, as type "${other.type}" and as type "${type}".`);
|
|
||||||
}
|
|
||||||
this.#installEntry(next, registered, hydrated);
|
|
||||||
}
|
}
|
||||||
}
|
const hydrated = this.#admitEntry(next, registered, admitted, value, unreadSlots);
|
||||||
|
const other = this.#heldUnderOtherType(next, type, hydrated.key);
|
||||||
const foreignByKey = new Map<string, string>();
|
|
||||||
for (const record of next.foreign) {
|
|
||||||
const first = foreignByKey.get(record.key);
|
|
||||||
if (first !== undefined) {
|
|
||||||
throw new Error(`Record ${record.key} is held by two stores, as schema "${first}" and as schema "${record.schemaName}".`);
|
|
||||||
}
|
|
||||||
foreignByKey.set(record.key, record.schemaName);
|
|
||||||
const other = [...this.#definitions.keys()].map((type) => next.entries.get(this.#entryId(type, record.key))).find((held) => held !== undefined);
|
|
||||||
if (other !== undefined) {
|
if (other !== undefined) {
|
||||||
throw new Error(`Record ${record.key} is held by two stores, as type "${other.type}" and as schema "${record.schemaName}".`);
|
throw new Error(`Record ${hydrated.key} is held by two stores, as type "${other.type}" and as type "${type}".`);
|
||||||
|
}
|
||||||
|
this.#installEntry(next, registered, hydrated);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const foreignByKey = new Map<string, string>();
|
||||||
|
for (const record of next.foreign) {
|
||||||
|
const first = foreignByKey.get(record.key);
|
||||||
|
if (first !== undefined) {
|
||||||
|
throw new Error(`Record ${record.key} is held by two stores, as schema "${first}" and as schema "${record.schemaName}".`);
|
||||||
|
}
|
||||||
|
foreignByKey.set(record.key, record.schemaName);
|
||||||
|
const other = [...this.#definitions.keys()].map((type) => next.entries.get(this.#entryId(type, record.key))).find((held) => held !== undefined);
|
||||||
|
if (other !== undefined) {
|
||||||
|
throw new Error(`Record ${record.key} is held by two stores, as type "${other.type}" and as schema "${record.schemaName}".`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Judged whole, once: the refusal names the least violation of the state this load would leave, whatever the
|
||||||
|
// order of the stores and of the entries in them.
|
||||||
|
refuseLeastIndexSharing(function* () {
|
||||||
|
for (const entry of next.entries.values()) {
|
||||||
|
for (const [alias, value] of entry.lookups.indexes) {
|
||||||
|
yield { type: entry.type, schemaName: entry.catalogEntry.schemaName, alias, value, key: entry.key };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}());
|
||||||
|
|
||||||
// Judged whole, once: the refusal names the least violation of the state this load would leave, whatever the
|
this.#hydrated = next;
|
||||||
// order of the stores and of the entries in them.
|
|
||||||
refuseLeastIndexSharing(function* () {
|
|
||||||
for (const entry of next.entries.values()) {
|
|
||||||
for (const [alias, value] of entry.lookups.indexes) {
|
|
||||||
yield { type: entry.type, schemaName: entry.catalogEntry.schemaName, alias, value, key: entry.key };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}());
|
|
||||||
|
|
||||||
this.#hydrated = next;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get<TDefinition extends AnyCultCacheDocumentDefinition>(
|
get<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
|
|
@ -513,12 +522,16 @@ export class CultCache {
|
||||||
return key ? this.get(definition, key) : undefined;
|
return key ? this.get(definition, key) : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The blind write: the record at the key becomes `value`, whatever it held, and the last writer wins. `update` is the
|
||||||
|
* conditional path, which changes a record from what the store holds now and re-runs on a lost race.
|
||||||
|
*/
|
||||||
put<TDefinition extends AnyCultCacheDocumentDefinition>(
|
put<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
definition: TDefinition,
|
definition: TDefinition,
|
||||||
key: string,
|
key: string,
|
||||||
value: CultCacheDocumentValue<TDefinition>,
|
value: CultCacheDocumentValue<TDefinition>,
|
||||||
): Promise<CultCacheDocumentValue<TDefinition>> {
|
): Promise<CultCacheDocumentValue<TDefinition>> {
|
||||||
return this.#serial(() => this.#putNow(definition, key, value));
|
return this.#serial(async () => (await this.#putNow(definition, key, value)).value);
|
||||||
}
|
}
|
||||||
|
|
||||||
putEnvelope<TDefinition extends AnyCultCacheDocumentDefinition>(
|
putEnvelope<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
|
|
@ -560,31 +573,68 @@ export class CultCache {
|
||||||
definition: TDefinition,
|
definition: TDefinition,
|
||||||
value: CultCacheDocumentValue<TDefinition>,
|
value: CultCacheDocumentValue<TDefinition>,
|
||||||
): Promise<CultCacheDocumentValue<TDefinition>> {
|
): Promise<CultCacheDocumentValue<TDefinition>> {
|
||||||
return this.#serial(() => {
|
return this.#serial(async () => {
|
||||||
this.#requireGlobal(definition);
|
this.#requireGlobal(definition);
|
||||||
return this.#putNow(definition, CultCache.GLOBAL_KEY, value);
|
return (await this.#putNow(definition, CultCache.GLOBAL_KEY, value)).value;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
update<TDefinition extends AnyCultCacheDocumentDefinition>(
|
/**
|
||||||
|
* Changes the record at the key from what the store holds now. `apply` may run up to `maxAttempts` times (default 5): each
|
||||||
|
* attempt gets a fresh copy of the record as the store holds it (undefined when absent), which it may change and return, and
|
||||||
|
* the result is written only if the record is still that version; a lost race runs `apply` again on the newer record. It must
|
||||||
|
* not write this cache or have side effects. Returning undefined writes nothing (`declined`). A lost race that outlasts
|
||||||
|
* `maxAttempts` returns `lostRace` and does not throw; a refusal, or a throw from `apply`, rejects and is not retried. The
|
||||||
|
* write is durable when the promise resolves. `document` is the cache's record after the outcome. `put` is the blind write.
|
||||||
|
*/
|
||||||
|
async update<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
definition: TDefinition,
|
definition: TDefinition,
|
||||||
key: string,
|
key: string,
|
||||||
updater: (
|
apply: (
|
||||||
current: CultCacheDocumentValue<TDefinition> | undefined,
|
current: CultCacheDocumentValue<TDefinition> | undefined,
|
||||||
) => CultCacheDocumentValue<TDefinition>,
|
) => CultCacheDocumentValue<TDefinition> | undefined | Promise<CultCacheDocumentValue<TDefinition> | undefined>,
|
||||||
): Promise<CultCacheDocumentValue<TDefinition>> {
|
options: CultCacheUpdateOptions = {},
|
||||||
return this.#serial(() => this.#putNow(definition, key, updater(this.get(definition, key))));
|
): Promise<CultCacheUpdateResult<CultCacheDocumentValue<TDefinition>>> {
|
||||||
|
if (typeof apply !== "function") {
|
||||||
|
throw new TypeError("CultCache update needs a function to apply.");
|
||||||
|
}
|
||||||
|
// undefined is the default; anything else must be an integer of at least 1, so null is refused.
|
||||||
|
const maxAttempts = options.maxAttempts === undefined ? 5 : options.maxAttempts;
|
||||||
|
if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
|
||||||
|
throw new RangeError(`CultCache update maxAttempts must be an integer of at least 1, not ${String(maxAttempts)}.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let attempt = 1; ; attempt++) {
|
||||||
|
const { expected, copy } = await this.#serial(() => this.#readForUpdate(definition, key));
|
||||||
|
// Outside the serial queue and the store's lock: apply may await, and may write this cache through other keys.
|
||||||
|
const next = await apply(copy as CultCacheDocumentValue<TDefinition> | undefined);
|
||||||
|
if (next === undefined || next === null) {
|
||||||
|
return { outcome: "declined", document: this.get(definition, key) };
|
||||||
|
}
|
||||||
|
const written = await this.#serial(() => this.#putNow(definition, key, next, expected));
|
||||||
|
if (written.outcome === "committed") {
|
||||||
|
return { outcome: "updated", document: this.get(definition, key) as CultCacheDocumentValue<TDefinition> };
|
||||||
|
}
|
||||||
|
if (attempt === maxAttempts) {
|
||||||
|
// Display only. A cache with no store has nothing to pull, and a pull would replace what it holds with nothing.
|
||||||
|
if (this.#stores.length > 0) {
|
||||||
|
await this.#serial(() => this.#pullNow());
|
||||||
|
}
|
||||||
|
return { outcome: "lostRace", document: this.get(definition, key) };
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
updateGlobal<TDefinition extends AnyCultCacheDocumentDefinition>(
|
/** `update` of the type's global record. */
|
||||||
|
async updateGlobal<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
definition: TDefinition,
|
definition: TDefinition,
|
||||||
updater: (
|
apply: (
|
||||||
current: CultCacheDocumentValue<TDefinition> | undefined,
|
current: CultCacheDocumentValue<TDefinition> | undefined,
|
||||||
) => CultCacheDocumentValue<TDefinition>,
|
) => CultCacheDocumentValue<TDefinition> | undefined | Promise<CultCacheDocumentValue<TDefinition> | undefined>,
|
||||||
): Promise<CultCacheDocumentValue<TDefinition>> {
|
options: CultCacheUpdateOptions = {},
|
||||||
return this.#serial(() =>
|
): Promise<CultCacheUpdateResult<CultCacheDocumentValue<TDefinition>>> {
|
||||||
this.#putNow(definition, CultCache.GLOBAL_KEY, updater(this.getGlobal(definition))),
|
this.#requireGlobal(definition);
|
||||||
);
|
return this.update(definition, CultCache.GLOBAL_KEY, apply, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
delete<TDefinition extends AnyCultCacheDocumentDefinition>(
|
delete<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
|
|
@ -724,11 +774,55 @@ export class CultCache {
|
||||||
this.#stores.push(candidate);
|
this.#stores.push(candidate);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One attempt's read for update: the record at the key as its home holds it now, the condition a commit must meet, and a copy of
|
||||||
|
// the record for apply to change. A store-backed cache asks its store, which reports the record and its version from one read;
|
||||||
|
// a cache with no store has only the held entry, whose identity is the condition and whose admitted payload is the copy.
|
||||||
|
async #readForUpdate(
|
||||||
|
definition: AnyCultCacheDocumentDefinition,
|
||||||
|
key: string,
|
||||||
|
): Promise<{ expected: WriteCondition; copy: unknown }> {
|
||||||
|
const registered = this.#requireDefinition(definition);
|
||||||
|
this.#requireGlobalKey(registered, key);
|
||||||
|
const type = definition.type;
|
||||||
|
this.#refuseLegacyConditional(type);
|
||||||
|
const home = this.#homeStore(type);
|
||||||
|
if (home === undefined) {
|
||||||
|
const held = this.#hydrated.entries.get(this.#entryId(type, key));
|
||||||
|
if (held === undefined && this.#heldUnderOtherType(this.#hydrated, type, key) !== undefined) {
|
||||||
|
throw new Error(`Record ${key} is not a "${type}"; update cannot copy it.`);
|
||||||
|
}
|
||||||
|
return { expected: { held }, copy: held ? registered.formatter.decode(this.#cloneBytes(held.payload)).value : undefined };
|
||||||
|
}
|
||||||
|
if (home.read === undefined || home.commitIf === undefined) {
|
||||||
|
throw new Error(`CultCache backing store for type "${type}" cannot write conditionally; update needs a store with read and commitIf.`);
|
||||||
|
}
|
||||||
|
const found = await home.read(key);
|
||||||
|
if (found === undefined) {
|
||||||
|
return { expected: { version: null }, copy: undefined };
|
||||||
|
}
|
||||||
|
if (this.#resolveDefinitionForEnvelope(found.envelope) !== registered) {
|
||||||
|
throw new Error(`Record ${key} is not a "${type}"; update cannot copy it.`);
|
||||||
|
}
|
||||||
|
return { expected: { version: found.version }, copy: registered.formatter.decode(this.#cloneBytes(found.envelope.payload)).value };
|
||||||
|
}
|
||||||
|
|
||||||
|
#putNow<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
|
definition: TDefinition,
|
||||||
|
key: string,
|
||||||
|
value: CultCacheDocumentValue<TDefinition>,
|
||||||
|
): Promise<{ outcome: "committed"; value: CultCacheDocumentValue<TDefinition> }>;
|
||||||
|
#putNow<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
|
definition: TDefinition,
|
||||||
|
key: string,
|
||||||
|
value: CultCacheDocumentValue<TDefinition>,
|
||||||
|
expected: WriteCondition,
|
||||||
|
): Promise<{ outcome: "committed"; value: CultCacheDocumentValue<TDefinition> } | { outcome: "mismatch" }>;
|
||||||
async #putNow<TDefinition extends AnyCultCacheDocumentDefinition>(
|
async #putNow<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||||
definition: TDefinition,
|
definition: TDefinition,
|
||||||
key: string,
|
key: string,
|
||||||
value: CultCacheDocumentValue<TDefinition>,
|
value: CultCacheDocumentValue<TDefinition>,
|
||||||
): Promise<CultCacheDocumentValue<TDefinition>> {
|
expected?: WriteCondition,
|
||||||
|
): Promise<{ outcome: "committed"; value: CultCacheDocumentValue<TDefinition> } | { outcome: "mismatch" }> {
|
||||||
const registered = this.#requireDefinition(definition);
|
const registered = this.#requireDefinition(definition);
|
||||||
const payload = registered.formatter.encode(value);
|
const payload = registered.formatter.encode(value);
|
||||||
const { value: parsed, unreadSlots } = registered.formatter.decode(payload) as { value: CultCacheDocumentValue<TDefinition>; unreadSlots?: NotedSlots };
|
const { value: parsed, unreadSlots } = registered.formatter.decode(payload) as { value: CultCacheDocumentValue<TDefinition>; unreadSlots?: NotedSlots };
|
||||||
|
|
@ -740,19 +834,55 @@ export class CultCache {
|
||||||
schemaId: registered.catalogEntry.schemaId,
|
schemaId: registered.catalogEntry.schemaId,
|
||||||
catalogEntry: registered.catalogEntry,
|
catalogEntry: registered.catalogEntry,
|
||||||
};
|
};
|
||||||
await this.#writeNow(registered, entry, parsed, unreadSlots);
|
const outcome = await this.#writeNow(registered, entry, parsed, unreadSlots, expected);
|
||||||
return parsed;
|
return outcome === "committed" ? { outcome, value: parsed } : { outcome };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validates the whole write (home, global singleton, name and index accessors) before the store
|
// Validates the whole write (home, global singleton, name and index accessors) before the store
|
||||||
// is touched; once the store accepts, applying to the cache has no fallible step.
|
// is touched; once the store accepts, applying to the cache has no fallible step.
|
||||||
async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown, unreadSlots?: NotedSlots): Promise<void> {
|
// With a condition, the write lands only if the record at the key is still the one the caller read: the store's version
|
||||||
|
// (commitIf), or for a cache with no store the held entry itself. Otherwise nothing is written or installed and the answer is
|
||||||
|
// "mismatch". Without one the write is blind, as put's.
|
||||||
|
async #writeNow(
|
||||||
|
registered: RegisteredDefinition,
|
||||||
|
entry: CultCacheEnvelope,
|
||||||
|
value: unknown,
|
||||||
|
unreadSlots?: NotedSlots,
|
||||||
|
expected?: WriteCondition,
|
||||||
|
): Promise<"committed" | "mismatch"> {
|
||||||
const home = this.#homeStore(entry.type);
|
const home = this.#homeStore(entry.type);
|
||||||
const retyped = this.#judge(registered, entry.key, entry);
|
const retyped = this.#judge(registered, entry.key, entry);
|
||||||
const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value, unreadSlots);
|
const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value, unreadSlots);
|
||||||
this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups);
|
this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups);
|
||||||
const legacy = this.#hydrated.legacyGlobals.get(entry.type);
|
const legacy = this.#hydrated.legacyGlobals.get(entry.type);
|
||||||
await home?.push(entry);
|
// A conditional write over a legacy global is refused here too (a pull while apply awaited can have loaded one): the condition
|
||||||
|
// was read from the global's own key, the record sits under its legacy key, and the retire below would delete it unseen.
|
||||||
|
if (expected !== undefined) {
|
||||||
|
this.#refuseLegacyConditional(entry.type);
|
||||||
|
}
|
||||||
|
if (expected !== undefined && "held" in expected) {
|
||||||
|
// The condition was fixed at read for a cache with no store. A store attached since has its own record at the key that
|
||||||
|
// this read never saw, so the held identity no longer judges the write: lose the race and read again.
|
||||||
|
if (home !== undefined || this.#hydrated.entries.get(this.#entryId(entry.type, entry.key)) !== expected.held) {
|
||||||
|
return "mismatch";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (expected !== undefined && "version" in expected) {
|
||||||
|
if (home?.commitIf === undefined) {
|
||||||
|
throw new Error(`CultCache backing store for type "${entry.type}" cannot write conditionally; update needs a store with read and commitIf.`);
|
||||||
|
}
|
||||||
|
const committed = await home.commitIf(entry, expected.version);
|
||||||
|
if (committed.outcome === "mismatch") {
|
||||||
|
return "mismatch";
|
||||||
|
}
|
||||||
|
hydrated.storedAt = committed.storedAt;
|
||||||
|
} else {
|
||||||
|
// A store that re-stamps a write (storedAt later than the record it replaces) reports the stamp; the cache holds that one.
|
||||||
|
const pushed = await home?.push(entry);
|
||||||
|
if (typeof pushed === "object" && pushed !== null && typeof pushed.storedAt === "string" && !Number.isNaN(Date.parse(pushed.storedAt))) {
|
||||||
|
hydrated.storedAt = pushed.storedAt;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (home && legacy) {
|
if (home && legacy) {
|
||||||
await this.#retireLegacyGlobal(home, legacy, entry);
|
await this.#retireLegacyGlobal(home, legacy, entry);
|
||||||
}
|
}
|
||||||
|
|
@ -762,6 +892,17 @@ export class CultCache {
|
||||||
this.#removeEntry(this.#hydrated, this.#definitions.get(retyped.type)!, retyped);
|
this.#removeEntry(this.#hydrated, this.#definitions.get(retyped.type)!, retyped);
|
||||||
}
|
}
|
||||||
this.#installEntry(this.#hydrated, registered, hydrated);
|
this.#installEntry(this.#hydrated, registered, hydrated);
|
||||||
|
return "committed";
|
||||||
|
}
|
||||||
|
|
||||||
|
// The one rule that a conditional write (update, whichever entry reaches it) never touches a global loaded from a legacy key:
|
||||||
|
// only putGlobal retires that key. Asked before apply runs and again at the write.
|
||||||
|
#refuseLegacyConditional(type: string): void {
|
||||||
|
if (this.#hydrated.legacyGlobals.has(type)) {
|
||||||
|
throw new Error(
|
||||||
|
`CultCache global "${type}" was loaded from a legacy key; write it once with putGlobal to retire its legacy key.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An envelope of the definition's type at the key, carrying nothing: what the judge asks about a write or a removal by that type.
|
// An envelope of the definition's type at the key, carrying nothing: what the judge asks about a write or a removal by that type.
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
import { lstatSync, readFileSync } from "node:fs";
|
import { lstatSync, readFileSync } from "node:fs";
|
||||||
import { mkdir } from "node:fs/promises";
|
import { mkdir } from "node:fs/promises";
|
||||||
import { dirname, resolve } from "node:path";
|
import { dirname, resolve } from "node:path";
|
||||||
|
|
@ -55,14 +56,42 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
||||||
return disk.envelopes;
|
return disk.envelopes;
|
||||||
}
|
}
|
||||||
|
|
||||||
async push(entry: CultCacheEnvelope): Promise<void> {
|
async push(entry: CultCacheEnvelope): Promise<{ storedAt: string }> {
|
||||||
await this.#enqueue(async () => {
|
return this.#enqueue(async () => {
|
||||||
const disk = this.#readDisk();
|
const disk = this.#readDisk();
|
||||||
this.#format = disk.format;
|
this.#format = disk.format;
|
||||||
const existing = disk.envelopes;
|
const existing = disk.envelopes;
|
||||||
requireOneTypePerKey(this.filePath, [entry], existing);
|
requireOneTypePerKey(this.filePath, [entry], existing);
|
||||||
const filtered = existing.filter((candidate) => candidate.key !== entry.key);
|
const [written] = await this.#writeAll([entry], existing);
|
||||||
await this.#writeAll([entry], filtered);
|
return { storedAt: written!.storedAt };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// The record at the key as the file holds it now, with its content version, from one read. Like pullAll it takes no lock: a
|
||||||
|
// reader sees a whole file (the commit renames one over the other), and the version it reports is the one commitIf judges.
|
||||||
|
async read(key: string): Promise<{ envelope: CultCacheEnvelope; version: string } | undefined> {
|
||||||
|
const envelope = readSingleFileStore(this.filePath).envelopes.find((candidate) => candidate.key === key);
|
||||||
|
return envelope === undefined ? undefined : { envelope, version: recordVersion(envelope) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writes `entry` the way push does, but only if the record at its key (of any type) still has `expectedVersion`; null expects
|
||||||
|
// none. The comparison and the write are one hold of the write lock, so no other writer lands between them.
|
||||||
|
async commitIf(
|
||||||
|
entry: CultCacheEnvelope,
|
||||||
|
expectedVersion: string | null,
|
||||||
|
): Promise<{ outcome: "committed"; storedAt: string } | { outcome: "mismatch" }> {
|
||||||
|
return this.#enqueue(async () => {
|
||||||
|
const disk = this.#readDisk();
|
||||||
|
this.#format = disk.format;
|
||||||
|
const existing = disk.envelopes;
|
||||||
|
const current = existing.find((candidate) => candidate.key === entry.key);
|
||||||
|
if ((current === undefined ? null : recordVersion(current)) !== expectedVersion) {
|
||||||
|
return { outcome: "mismatch" as const };
|
||||||
|
}
|
||||||
|
|
||||||
|
requireOneTypePerKey(this.filePath, [entry], existing);
|
||||||
|
const [written] = await this.#writeAll([entry], existing);
|
||||||
|
return { outcome: "committed" as const, storedAt: written!.storedAt };
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -126,13 +155,67 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise<void> {
|
// Writes the supplied records onto the arrived ones and returns the supplied records as written. A supplied record whose
|
||||||
|
// schema id and payload bytes differ from the arrived one it replaces is never stamped earlier than the record it replaces, so a content version that includes storedAt
|
||||||
|
// cannot repeat across writes (variants:ruling:storedat-on-change): one not later than the arrived stamp is written one
|
||||||
|
// millisecond after it.
|
||||||
|
async #writeAll(supplied: CultCacheEnvelope[], arrived: CultCacheEnvelope[]): Promise<CultCacheEnvelope[]> {
|
||||||
|
const arrivedByKey = new Map(arrived.map((entry) => [entry.key, entry]));
|
||||||
|
const written = supplied.map((entry) => {
|
||||||
|
const prior = arrivedByKey.get(entry.key);
|
||||||
|
if (prior === undefined) {
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
// A rewrite of the identical record (same schema id and payload bytes) keeps the stamp and so the version it has; the
|
||||||
|
// writer's type and catalog entry still go down with it.
|
||||||
|
if (sameContent(entry, prior)) {
|
||||||
|
return { ...entry, storedAt: prior.storedAt };
|
||||||
|
}
|
||||||
|
return { ...entry, storedAt: laterStoredAt(entry.storedAt, prior.storedAt) };
|
||||||
|
});
|
||||||
await mkdir(dirname(this.filePath), { recursive: true });
|
await mkdir(dirname(this.filePath), { recursive: true });
|
||||||
|
|
||||||
await commitStoreFile(this.filePath, encode(encodeSnapshot(supplied, arrived, this.#format)));
|
await commitStoreFile(this.filePath, encode(encodeSnapshot(written, arrived, this.#format)));
|
||||||
|
return written;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The content version of a stored record: the lowercase hex SHA-256 of its persisted encoding, the MessagePack array
|
||||||
|
// [key, schemaId, storedAt, payload] as the store writes it (the C# reference's StoredVersion). Private to this module so
|
||||||
|
// node:crypto stays out of the browser-safe cache.
|
||||||
|
export function recordVersion(entry: CultCacheEnvelope): string {
|
||||||
|
return createHash("sha256")
|
||||||
|
.update(encode([entry.key, schemaIdForEnvelope(entry), entry.storedAt, documentPayloadBytes(entry.payload)]))
|
||||||
|
.digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameContent(a: CultCacheEnvelope, b: CultCacheEnvelope): boolean {
|
||||||
|
if (schemaIdForEnvelope(a) !== schemaIdForEnvelope(b)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const x = documentPayloadBytes(a.payload);
|
||||||
|
const y = documentPayloadBytes(b.payload);
|
||||||
|
return x.length === y.length && x.every((byte, i) => byte === y[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// `storedAt` if it is later than `prior`, else `prior` plus one millisecond. A stamp that does not parse is not later; a prior
|
||||||
|
// that does not parse leaves nothing to be later than.
|
||||||
|
function laterStoredAt(storedAt: string, prior: string): string {
|
||||||
|
const priorMs = Date.parse(prior);
|
||||||
|
if (Number.isNaN(priorMs)) {
|
||||||
|
return storedAt;
|
||||||
|
}
|
||||||
|
const storedMs = Date.parse(storedAt);
|
||||||
|
if (!(Number.isNaN(storedMs) || storedMs <= priorMs)) {
|
||||||
|
return storedAt;
|
||||||
|
}
|
||||||
|
// The stamp is ISO-8601 with a four-digit year; one past 9999-12-31T23:59:59.999Z would print as an extended year.
|
||||||
|
if (priorMs + 1 > Date.UTC(9999, 11, 31, 23, 59, 59, 999)) {
|
||||||
|
throw new RangeError(`CultCache cannot stamp a write later than the record it replaces (${prior}): the stamp would leave the four-digit years.`);
|
||||||
|
}
|
||||||
|
return new Date(priorMs + 1).toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
// A store holds a key once, and the reader refuses a file that holds it twice, so a write that would put a second type under a
|
// A store holds a key once, and the reader refuses a file that holds it twice, so a write that would put a second type under a
|
||||||
// key (against a record on disk, or within its own batch) is refused before anything is written. Retyping a record is a delete
|
// key (against a record on disk, or within its own batch) is refused before anything is written. Retyping a record is a delete
|
||||||
// of the old type followed by a put of the new one.
|
// of the old type followed by a put of the new one.
|
||||||
|
|
|
||||||
|
|
@ -218,6 +218,8 @@ export async function withStoreWriteLock<T>(
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
// A link may have been put at the path while the lock was awaited; the look above only saw the path before.
|
||||||
|
await refuseLinkedStore(storePath);
|
||||||
return await operation();
|
return await operation();
|
||||||
} finally {
|
} finally {
|
||||||
release();
|
release();
|
||||||
|
|
|
||||||
33
packages/cultcache-ts/src/stored-at.ts
Normal file
33
packages/cultcache-ts/src/stored-at.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
// The one shape a `storedAt` stamp has (tests/vectors/stored-at-stamp/README.md; reference `parse_stored_at` in cultcache-py): an
|
||||||
|
// RFC 3339 date-time `YYYY-MM-DDThh:mm:ss`, an optional fraction of 1 to 9 digits, and `Z` or `+hh:mm` / `-hh:mm` (hours 00-23,
|
||||||
|
// minutes 00-59). A real calendar date, seconds 00-59, and an instant within years 0001-9999. Checked here, not left to
|
||||||
|
// `Date.parse`, which accepts shapes this one does not and whose accepted set differs by engine.
|
||||||
|
const STAMP = /^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]{1,9})?(?:Z|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))$/;
|
||||||
|
|
||||||
|
const MS_PER_MINUTE = 60_000;
|
||||||
|
|
||||||
|
/** True when `text` is a string that is a stamp. Anything else (missing, empty, wrong type, other shapes) is not. */
|
||||||
|
export function isStoredAtStamp(text: unknown): text is string {
|
||||||
|
if (typeof text !== "string") return false;
|
||||||
|
const shape = STAMP.exec(text);
|
||||||
|
if (shape === null) return false;
|
||||||
|
const [year, month, day, hour, minute, second] = shape.slice(1, 7).map(Number) as [number, number, number, number, number, number];
|
||||||
|
const leap = (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0;
|
||||||
|
const daysInMonth = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month - 1];
|
||||||
|
if (year < 1 || month < 1 || month > 12 || day < 1 || day > (daysInMonth as number) || hour > 23 || minute > 59 || second > 59) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// The UTC instant must stay within years 0001-9999, so an offset can push a boundary date out of range.
|
||||||
|
const local = new Date(0);
|
||||||
|
local.setUTCFullYear(year, month - 1, day);
|
||||||
|
local.setUTCHours(hour, minute, second, 0);
|
||||||
|
const offsetMinutes = shape[7] === undefined ? 0 : (shape[7] === "-" ? -1 : 1) * (Number(shape[8]) * 60 + Number(shape[9]));
|
||||||
|
const utcMs = local.getTime() - offsetMinutes * MS_PER_MINUTE;
|
||||||
|
const first = new Date(0);
|
||||||
|
first.setUTCFullYear(1, 0, 1);
|
||||||
|
first.setUTCHours(0, 0, 0, 0);
|
||||||
|
const last = new Date(0);
|
||||||
|
last.setUTCFullYear(9999, 11, 31);
|
||||||
|
last.setUTCHours(23, 59, 59, 999);
|
||||||
|
return utcMs >= first.getTime() && utcMs <= last.getTime();
|
||||||
|
}
|
||||||
|
|
@ -125,9 +125,34 @@ export interface PushAllOptions {
|
||||||
|
|
||||||
export interface CacheBackingStore {
|
export interface CacheBackingStore {
|
||||||
pullAll(): Promise<CultCacheEnvelope[]>;
|
pullAll(): Promise<CultCacheEnvelope[]>;
|
||||||
push(entry: CultCacheEnvelope): Promise<void>;
|
/** A store that re-stamps the write returns the storedAt it wrote; the cache then holds that stamp, not its own. */
|
||||||
|
push(entry: CultCacheEnvelope): Promise<void | { storedAt: string }>;
|
||||||
delete(entry: CultCacheEnvelope): Promise<void>;
|
delete(entry: CultCacheEnvelope): Promise<void>;
|
||||||
pushAll?(entries: CultCacheEnvelope[], options?: PushAllOptions): Promise<void>;
|
pushAll?(entries: CultCacheEnvelope[], options?: PushAllOptions): Promise<void>;
|
||||||
|
/**
|
||||||
|
* The record at the key as the store holds it now, with its content version, from one read; undefined when the store holds
|
||||||
|
* none. The version is the store's own: the cache carries it from `read` to `commitIf` and never computes it.
|
||||||
|
*/
|
||||||
|
read?(key: string): Promise<{ envelope: CultCacheEnvelope; version: string } | undefined>;
|
||||||
|
/**
|
||||||
|
* Writes the entry only if the record at its key still has `expectedVersion` (null: no record), judged and written under the
|
||||||
|
* store's write lock. A mismatch writes nothing. A commit returns the storedAt actually written.
|
||||||
|
*/
|
||||||
|
commitIf?(
|
||||||
|
entry: CultCacheEnvelope,
|
||||||
|
expectedVersion: string | null,
|
||||||
|
): Promise<{ outcome: "committed"; storedAt: string } | { outcome: "mismatch" }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The outcome of `CultCache.update`: what `apply` wrote, declined to write, or lost to a rival writer. */
|
||||||
|
export type CultCacheUpdateResult<TValue> =
|
||||||
|
| { outcome: "updated"; document: TValue }
|
||||||
|
| { outcome: "declined"; document: TValue | undefined }
|
||||||
|
| { outcome: "lostRace"; document: TValue | undefined };
|
||||||
|
|
||||||
|
export interface CultCacheUpdateOptions {
|
||||||
|
/** How many times `apply` may run before a lost race is returned as `lostRace`. An integer of at least 1; default 5. */
|
||||||
|
maxAttempts?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CultCacheStoreRegistration {
|
export interface CultCacheStoreRegistration {
|
||||||
|
|
|
||||||
|
|
@ -2106,7 +2106,7 @@ async function readTsInteropStore(file: string): Promise<InteropNote> {
|
||||||
|
|
||||||
async function buildInteropPeers(): Promise<void> {
|
async function buildInteropPeers(): Promise<void> {
|
||||||
if (!(await exists(rustInteropBinary))) {
|
if (!(await exists(rustInteropBinary))) {
|
||||||
await execAsync(`"${cargoCommand}" build --quiet --example cultcache_interop`, {
|
await execAsync(`"${cargoCommand}" build --locked --quiet --example cultcache_interop`, {
|
||||||
cwd: cultcacheRsRoot,
|
cwd: cultcacheRsRoot,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,21 @@ async function firstCall(path: string) {
|
||||||
}
|
}
|
||||||
// snippet:end
|
// snippet:end
|
||||||
|
|
||||||
|
// The update snippet, which docs/start/README.md shows after the first call and which uses its definitions.
|
||||||
|
// snippet:update:begin
|
||||||
|
async function restock(path: string) {
|
||||||
|
const cache = CultCache.builder()
|
||||||
|
.withRegistry(registry)
|
||||||
|
.withGenericStore(new SingleFileMessagePackBackingStore(path))
|
||||||
|
.build();
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
return cache.update(itemDocument, "item:potion", (item) => {
|
||||||
|
const current = item ?? { name: "Potion", category: "Consumable", value: 0 };
|
||||||
|
return { ...current, value: current.value + 10 };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// snippet:update:end
|
||||||
|
|
||||||
test("define, store and read back a typed document", async () => {
|
test("define, store and read back a typed document", async () => {
|
||||||
const dir = await mkdtemp(join(tmpdir(), "cultcache-first-call-"));
|
const dir = await mkdtemp(join(tmpdir(), "cultcache-first-call-"));
|
||||||
try {
|
try {
|
||||||
|
|
@ -57,13 +72,35 @@ test("define, store and read back a typed document", async () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test("docs show the tested snippet", async () => {
|
test("update with retry: restock twice reads value 20 through a second cache", async () => {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "cultcache-first-call-"));
|
||||||
|
try {
|
||||||
|
const path = join(dir, "restock.cc");
|
||||||
|
assert.equal((await restock(path)).outcome, "updated");
|
||||||
|
assert.equal((await restock(path)).outcome, "updated");
|
||||||
|
const reopened = CultCache.builder()
|
||||||
|
.withRegistry(registry)
|
||||||
|
.withGenericStore(new SingleFileMessagePackBackingStore(path))
|
||||||
|
.build();
|
||||||
|
await reopened.pullAllBackingStores();
|
||||||
|
assert.equal(reopened.get(itemDocument, "item:potion")?.value, 20);
|
||||||
|
} finally {
|
||||||
|
await rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// The docs show the region between `// snippet:<region>begin` and `end` markers verbatim, the package import aside.
|
||||||
|
async function assertDocsShow(region: "" | "update:") {
|
||||||
// Compiled to dist-test/test/first-call.test.js, so the package root is two levels up.
|
// Compiled to dist-test/test/first-call.test.js, so the package root is two levels up.
|
||||||
const source = await readFile(join(__dirname, "..", "..", "test", "first-call.test.ts"), "utf8");
|
const source = await readFile(join(__dirname, "..", "..", "test", "first-call.test.ts"), "utf8");
|
||||||
const snippet = source
|
const snippet = source
|
||||||
.split("// snippet:begin\n")[1]
|
.split(`// snippet:${region}begin\n`)[1]
|
||||||
.split("// snippet:end\n")[0]
|
.split(`// snippet:${region}end\n`)[0]
|
||||||
.replace('"../src/index"', '"@gamecult/cultcache-ts"');
|
.replace('"../src/index"', '"@gamecult/cultcache-ts"');
|
||||||
const docs = (await readFile(join(__dirname, "..", "..", "..", "..", "docs", "start", "README.md"), "utf8")).replace(/\r\n/g, "\n");
|
const docs = (await readFile(join(__dirname, "..", "..", "..", "..", "docs", "start", "README.md"), "utf8")).replace(/\r\n/g, "\n");
|
||||||
assert.ok(docs.includes("```ts\n" + snippet + "```"), "docs/start/README.md no longer shows test/first-call.test.ts");
|
assert.ok(docs.includes("```ts\n" + snippet + "```"), `docs/start/README.md no longer shows the ${region || "first-call "}region of test/first-call.test.ts`);
|
||||||
});
|
}
|
||||||
|
|
||||||
|
test("docs show the tested snippet", () => assertDocsShow(""));
|
||||||
|
|
||||||
|
test("docs show the tested update snippet", () => assertDocsShow("update:"));
|
||||||
|
|
|
||||||
175
packages/cultcache-ts/test/store-conditional.test.ts
Normal file
175
packages/cultcache-ts/test/store-conditional.test.ts
Normal file
|
|
@ -0,0 +1,175 @@
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
|
import { copyFile, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import test from "node:test";
|
||||||
|
|
||||||
|
import { decode, encode } from "@msgpack/msgpack";
|
||||||
|
|
||||||
|
import { documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
||||||
|
import { SingleFileMessagePackBackingStore, readSingleFileStore, recordVersion } from "../src/single-file-messagepack-backing-store";
|
||||||
|
import { schemaIdForEnvelope } from "../src/store-format";
|
||||||
|
import type { CultCacheEnvelope } from "../src/types";
|
||||||
|
|
||||||
|
// The store owns the content version and the conditional verdict (variants:ruling:write-set-content-version-declared-ownership):
|
||||||
|
// read reports a record with its version, commitIf writes only while the version holds, and a write that replaces a record
|
||||||
|
// is stamped later than it (variants:ruling:storedat-on-change).
|
||||||
|
|
||||||
|
function findVectors(from: string): string {
|
||||||
|
for (let dir = from; ; dir = resolve(dir, "..")) {
|
||||||
|
if (existsSync(join(dir, "tests", "vectors", "content-version"))) return join(dir, "tests", "vectors", "content-version");
|
||||||
|
if (resolve(dir, "..") === dir) throw new Error("tests/vectors/content-version not found above the test.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry = (key: string, byte: number, storedAt = "2026-01-01T00:00:00.000Z", type = "t"): CultCacheEnvelope => ({
|
||||||
|
key,
|
||||||
|
type,
|
||||||
|
payload: documentPayloadFromBytes(Uint8Array.of(0x91, byte)),
|
||||||
|
storedAt,
|
||||||
|
});
|
||||||
|
|
||||||
|
async function scratch(): Promise<string> {
|
||||||
|
return mkdtemp(join(tmpdir(), "cultcache-conditional-"));
|
||||||
|
}
|
||||||
|
|
||||||
|
test("the content version of every record of the C#-written store equals the version the reference read", async () => {
|
||||||
|
const vectors = findVectors(__dirname);
|
||||||
|
const rows = decode(readFileSync(join(vectors, "versions.msgpack"))) as Array<[string, string]>;
|
||||||
|
assert.equal(rows.length, 2);
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "store.cc");
|
||||||
|
await copyFile(join(vectors, "store.cc"), path);
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
for (const [key, version] of rows) {
|
||||||
|
const found = await store.read(key);
|
||||||
|
assert.equal(found?.version, version, `read(${key}).version`);
|
||||||
|
assert.equal(recordVersion(found!.envelope), version, `recordVersion(${key})`);
|
||||||
|
}
|
||||||
|
assert.equal(await store.read("absent"), undefined);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the version is the SHA-256 of the record's persisted encoding and changes with every field of it", async () => {
|
||||||
|
const base = entry("k", 1);
|
||||||
|
const hex = (e: CultCacheEnvelope) =>
|
||||||
|
createHash("sha256").update(encode([e.key, schemaIdForEnvelope(e), e.storedAt, documentPayloadBytes(e.payload)])).digest("hex");
|
||||||
|
assert.equal(recordVersion(base), hex(base));
|
||||||
|
const versions = new Set([base, entry("k", 2), entry("k", 1, "2026-01-01T00:00:00.001Z"), entry("j", 1), { ...base, schemaId: "sha256:other" }].map(recordVersion));
|
||||||
|
assert.equal(versions.size, 5);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("commitIf writes nothing when the version does not hold, and commits when it does", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "store.cc");
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
// null expects no record: it commits over an absent key and mismatches over a present one.
|
||||||
|
assert.deepEqual(await store.commitIf(entry("k", 1), "not-a-version"), { outcome: "mismatch" });
|
||||||
|
assert.equal(existsSync(path), false, "a mismatch creates no file");
|
||||||
|
assert.equal((await store.commitIf(entry("k", 1), null)).outcome, "committed");
|
||||||
|
const first = (await store.read("k"))!;
|
||||||
|
const before = await readFile(path);
|
||||||
|
assert.deepEqual(await store.commitIf(entry("k", 2), null), { outcome: "mismatch" });
|
||||||
|
assert.deepEqual(await store.commitIf(entry("k", 2), "0".repeat(64)), { outcome: "mismatch" });
|
||||||
|
assert.deepEqual([...(await readFile(path))], [...before], "a mismatch leaves the bytes identical");
|
||||||
|
assert.equal((await store.commitIf(entry("k", 2), first.version)).outcome, "committed");
|
||||||
|
const second = (await store.read("k"))!;
|
||||||
|
assert.notEqual(second.version, first.version);
|
||||||
|
assert.deepEqual([...documentPayloadBytes(second.envelope.payload)], [0x91, 2]);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("commitIf judges the record at the key whatever its type, and a record of another key does not matter", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
||||||
|
await store.push(entry("a", 1, undefined, "u"));
|
||||||
|
await store.push(entry("b", 1));
|
||||||
|
const other = (await store.read("a"))!;
|
||||||
|
assert.deepEqual(await store.commitIf(entry("b", 2), other.version), { outcome: "mismatch" });
|
||||||
|
assert.equal((await store.commitIf(entry("b", 2), recordVersion((await store.read("b"))!.envelope))).outcome, "committed");
|
||||||
|
assert.deepEqual([...documentPayloadBytes((await store.read("a"))!.envelope.payload)], [0x91, 1], "the record at another key is untouched");
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a write that replaces a record is stored strictly later than it, whatever the write's own storedAt", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
||||||
|
const at = "2026-03-04T05:06:07.100Z";
|
||||||
|
await store.push(entry("k", 1, at));
|
||||||
|
// Not later (equal, earlier, unparseable) is written one millisecond after the record it replaces, by push, pushAll and commitIf.
|
||||||
|
await store.push(entry("k", 2, at));
|
||||||
|
assert.equal((await store.read("k"))!.envelope.storedAt, "2026-03-04T05:06:07.101Z");
|
||||||
|
await store.push(entry("k", 3, "2020-01-01T00:00:00.000Z"));
|
||||||
|
assert.equal((await store.read("k"))!.envelope.storedAt, "2026-03-04T05:06:07.102Z");
|
||||||
|
await store.pushAll([entry("k", 4, "not a time")]);
|
||||||
|
assert.equal((await store.read("k"))!.envelope.storedAt, "2026-03-04T05:06:07.103Z");
|
||||||
|
const committed = await store.commitIf(entry("k", 5, "2020-01-01T00:00:00.000Z"), (await store.read("k"))!.version);
|
||||||
|
assert.deepEqual(committed, { outcome: "committed", storedAt: "2026-03-04T05:06:07.104Z" });
|
||||||
|
assert.equal((await store.read("k"))!.envelope.storedAt, "2026-03-04T05:06:07.104Z");
|
||||||
|
// A later stamp is kept as it is, and a key with no record before it is stamped as it is.
|
||||||
|
await store.push(entry("k", 6, "2030-01-01T00:00:00.000Z"));
|
||||||
|
await store.push(entry("fresh", 1, "2000-01-01T00:00:00.000Z"));
|
||||||
|
assert.equal((await store.read("k"))!.envelope.storedAt, "2030-01-01T00:00:00.000Z");
|
||||||
|
assert.equal((await store.read("fresh"))!.envelope.storedAt, "2000-01-01T00:00:00.000Z");
|
||||||
|
assert.equal(readSingleFileStore(join(directory, "store.cc")).envelopes.length, 2);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a replacing write that would stamp past year 9999 is refused and writes nothing", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
||||||
|
const last = "9999-12-31T23:59:59.999Z";
|
||||||
|
await store.push(entry("k", 1, last));
|
||||||
|
await assert.rejects(() => store.push(entry("k", 2, last)), RangeError);
|
||||||
|
assert.equal((await store.read("k"))!.envelope.storedAt, last);
|
||||||
|
assert.deepEqual([...documentPayloadBytes((await store.read("k"))!.envelope.payload)], [0x91, 1]);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a write that restores an earlier value does not restore its version", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const store = new SingleFileMessagePackBackingStore(join(directory, "store.cc"));
|
||||||
|
const at = "2026-03-04T05:06:07.100Z";
|
||||||
|
await store.push(entry("k", 1, at));
|
||||||
|
const first = (await store.read("k"))!.version;
|
||||||
|
await store.push(entry("k", 5, at));
|
||||||
|
await store.push(entry("k", 1, at));
|
||||||
|
assert.notEqual((await store.read("k"))!.version, first, "an A-B-A within one millisecond keeps no version");
|
||||||
|
assert.deepEqual(await store.commitIf(entry("k", 9, at), first), { outcome: "mismatch" });
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("two stores on one file never both commit against the same version", async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const path = join(directory, "store.cc");
|
||||||
|
const left = new SingleFileMessagePackBackingStore(path);
|
||||||
|
const right = new SingleFileMessagePackBackingStore(path);
|
||||||
|
await left.push(entry("k", 0));
|
||||||
|
const version = (await left.read("k"))!.version;
|
||||||
|
const outcomes = await Promise.all(Array.from({ length: 6 }, (_, i) => (i % 2 === 0 ? left : right).commitIf(entry("k", i + 1), version)));
|
||||||
|
assert.equal(outcomes.filter((o) => o.outcome === "committed").length, 1);
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
@ -417,6 +417,43 @@ test("a second writer in this process waits for the store lock and runs after re
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("a link put while a writer waits on the store lock is refused once it holds the lock", { skip: process.platform === "win32" }, async () => {
|
||||||
|
const directory = await scratch();
|
||||||
|
try {
|
||||||
|
const target = join(directory, "target.cc");
|
||||||
|
await new SingleFileMessagePackBackingStore(target).push(noteEnvelope("a"));
|
||||||
|
const before = await readFile(target);
|
||||||
|
const writes: Array<[string, (store: SingleFileMessagePackBackingStore) => Promise<void>]> = [
|
||||||
|
["push", async (store) => void (await store.push(noteEnvelope("b")))],
|
||||||
|
["pushAll", (store) => store.pushAll([noteEnvelope("b")])],
|
||||||
|
["delete", (store) => store.delete(noteEnvelope("a"))],
|
||||||
|
];
|
||||||
|
const missing = join(directory, "missing.cc");
|
||||||
|
for (const [name, write, linkTo] of writes.flatMap(([n, w]) => [[n, w, target], [`${n} (dangling)`, w, missing]] as const)) {
|
||||||
|
const link = join(directory, `${name.replace(/\W+/g, "-")}.cc`);
|
||||||
|
let writer: Promise<void> | undefined;
|
||||||
|
await withStoreWriteLock(link, async () => {
|
||||||
|
writer = write(new SingleFileMessagePackBackingStore(link));
|
||||||
|
writer.catch(() => undefined);
|
||||||
|
await sleepFor(150);
|
||||||
|
await fsp.symlink(linkTo, link); // after the writer's pre-lock look, while it waits
|
||||||
|
});
|
||||||
|
await assert.rejects(writer!, (error: unknown) => error instanceof PackageStoreLinkedError, name);
|
||||||
|
assert.ok((await fsp.lstat(link)).isSymbolicLink(), `${name}: the link was replaced`);
|
||||||
|
assert.deepEqual(await readFile(target), before, `${name}: the target changed`);
|
||||||
|
await assert.rejects(fsp.lstat(missing), { code: "ENOENT" }, `${name}: the dangling target was created`);
|
||||||
|
await rm(link); // the link is refused before the lock, so remove it to probe the lock
|
||||||
|
assert.equal(await withStoreWriteLock(link, async () => "free", { timeoutMs: 200 }), "free", `${name}: the lock is held`);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
function noteEnvelope(key: string): CultCacheEnvelope {
|
||||||
|
return { key, type: "note", payload: documentPayloadFromBytes(Uint8Array.of(0x91, 1)), storedAt: "2026-10-20T00:00:00Z" };
|
||||||
|
}
|
||||||
|
|
||||||
test("the store lock gives up after its wait bound and names the lock path", async () => {
|
test("the store lock gives up after its wait bound and names the lock path", async () => {
|
||||||
const directory = await scratch();
|
const directory = await scratch();
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
20
packages/cultcache-ts/test/stored-at-stamp.test.ts
Normal file
20
packages/cultcache-ts/test/stored-at-stamp.test.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
// isStoredAtStamp agrees with every row of tests/vectors/stored-at-stamp/stamps.msgpack, the one shape a peer's storedAt must parse as.
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import test from "node:test";
|
||||||
|
import { decode } from "@msgpack/msgpack";
|
||||||
|
import { isStoredAtStamp } from "../src";
|
||||||
|
|
||||||
|
// dist-test/test -> the repository root.
|
||||||
|
export const STAMP_VECTOR = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "stored-at-stamp", "stamps.msgpack");
|
||||||
|
|
||||||
|
test("isStoredAtStamp answers every row of the stored-at-stamp vector", () => {
|
||||||
|
const rows = decode(readFileSync(STAMP_VECTOR)) as [string, boolean][];
|
||||||
|
assert.ok(rows.length > 20, "the vector was read");
|
||||||
|
for (const [text, parses] of rows) assert.equal(isStoredAtStamp(text), parses, JSON.stringify(text));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a stamp is a string: anything else is not one", () => {
|
||||||
|
for (const value of [undefined, null, 0, 1_700_000_000_000, true, {}, [], new Date()]) assert.equal(isStoredAtStamp(value), false);
|
||||||
|
});
|
||||||
|
|
@ -7,6 +7,7 @@ import { z } from "zod";
|
||||||
import { codecs } from "../src";
|
import { codecs } from "../src";
|
||||||
import type { Codec } from "../src/codec";
|
import type { Codec } from "../src/codec";
|
||||||
import { defineDocumentType, documentPayloadFromBytes } from "../src/document";
|
import { defineDocumentType, documentPayloadFromBytes } from "../src/document";
|
||||||
|
import { recordVersion } from "../src/single-file-messagepack-backing-store";
|
||||||
import type { CacheBackingStore, CultCacheEnvelope } from "../src/types";
|
import type { CacheBackingStore, CultCacheEnvelope } from "../src/types";
|
||||||
|
|
||||||
// The union-surface fixture: the codecs, schema and value tests/vectors/union-parity was written from, shared by the
|
// The union-surface fixture: the codecs, schema and value tests/vectors/union-parity was written from, shared by the
|
||||||
|
|
@ -235,6 +236,16 @@ export function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & {
|
||||||
async pullAll() { return entries.map((entry) => ({ ...entry })); },
|
async pullAll() { return entries.map((entry) => ({ ...entry })); },
|
||||||
async push(entry) { entries.push(entry); },
|
async push(entry) { entries.push(entry); },
|
||||||
async delete() {},
|
async delete() {},
|
||||||
|
async read(key) {
|
||||||
|
const envelope = [...entries].reverse().find((entry) => entry.key === key);
|
||||||
|
return envelope === undefined ? undefined : { envelope, version: recordVersion(envelope) };
|
||||||
|
},
|
||||||
|
async commitIf(entry, expectedVersion) {
|
||||||
|
const current = [...entries].reverse().find((candidate) => candidate.key === entry.key);
|
||||||
|
if ((current === undefined ? null : recordVersion(current)) !== expectedVersion) return { outcome: "mismatch" };
|
||||||
|
entries.push(entry);
|
||||||
|
return { outcome: "committed", storedAt: entry.storedAt };
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
600
packages/cultcache-ts/test/update.test.ts
Normal file
600
packages/cultcache-ts/test/update.test.ts
Normal file
|
|
@ -0,0 +1,600 @@
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import test from "node:test";
|
||||||
|
import { setImmediate as tick } from "node:timers/promises";
|
||||||
|
|
||||||
|
import { decode, encode } from "@msgpack/msgpack";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
import { CultCache } from "../src/cult-cache";
|
||||||
|
import { defineDocumentRegistry, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
||||||
|
import { SingleFileMessagePackBackingStore, readSingleFileStore } from "../src/single-file-messagepack-backing-store";
|
||||||
|
import { SchemaConflictError, schemaIdForEnvelope } from "../src/store-format";
|
||||||
|
import type { CacheBackingStore } from "../src/types";
|
||||||
|
import { envelopeOf, memoryStore, read, surfaceDocument } from "./union-surface-fixture";
|
||||||
|
|
||||||
|
// CultCache.update: read the record and its content version from the store, apply a function to a copy, commit only if the
|
||||||
|
// version still holds, run again on a lost race (variants:cut_spec:cut-update-with-retry-ts.r1; the C# tests in
|
||||||
|
// tests/GameCult.Caching.Tests/UpdateTests.cs are the parity).
|
||||||
|
|
||||||
|
const counterDocument = defineDocumentType({
|
||||||
|
type: "update.counter",
|
||||||
|
schemaId: "update.counter.v1",
|
||||||
|
schemaName: "update.counter",
|
||||||
|
schemaVersion: "update.counter.v1",
|
||||||
|
schema: z.object({ value: z.number().int() }),
|
||||||
|
});
|
||||||
|
const tallyDocument = defineDocumentType({
|
||||||
|
type: "update.tally",
|
||||||
|
schemaId: "update.tally.v1",
|
||||||
|
schemaName: "update.tally",
|
||||||
|
schemaVersion: "update.tally.v1",
|
||||||
|
schema: z.object({ count: z.number().int() }),
|
||||||
|
});
|
||||||
|
const registry = defineDocumentRegistry(counterDocument, tallyDocument);
|
||||||
|
const settingsFields = { schemaName: "update.settings", schemaVersion: "update.settings.v1", schemaId: "update.settings.v1", schema: z.object({ theme: z.string() }) };
|
||||||
|
const settingsDocument = defineDocumentType({ type: "update.settings", ...settingsFields, global: true });
|
||||||
|
const plainSettingsDocument = defineDocumentType({ type: "update.settings", ...settingsFields });
|
||||||
|
|
||||||
|
const KEY = "counter";
|
||||||
|
|
||||||
|
async function withDirectory<T>(body: (path: string) => Promise<T>): Promise<T> {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "cultcache-update-"));
|
||||||
|
try {
|
||||||
|
return await body(join(directory, "store.cc"));
|
||||||
|
} finally {
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type Registry = Parameters<ReturnType<typeof CultCache.builder>["withRegistry"]>[0];
|
||||||
|
|
||||||
|
async function open(path: string, documents: Registry = registry): Promise<CultCache> {
|
||||||
|
const cache = CultCache.builder().withRegistry(documents).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
return cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the file holds at the key, read without a cache.
|
||||||
|
function stored(path: string, key = KEY): number | undefined {
|
||||||
|
const envelope = readSingleFileStore(path).envelopes.find((candidate) => candidate.key === key);
|
||||||
|
return envelope === undefined ? undefined : (decode(documentPayloadBytes(envelope.payload)) as { value: number }).value;
|
||||||
|
}
|
||||||
|
|
||||||
|
const plus = (n: number) => (current: { value: number } | undefined) => ({ value: (current?.value ?? 0) + n });
|
||||||
|
|
||||||
|
test("two writers updating lose no increment", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [left, right] = [await open(path), await open(path)];
|
||||||
|
let calls = 0;
|
||||||
|
const loop = async (cache: CultCache) => {
|
||||||
|
const outcomes: string[] = [];
|
||||||
|
for (let i = 0; i < 50; i++) {
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
calls++;
|
||||||
|
await tick();
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
}, { maxAttempts: 50 });
|
||||||
|
outcomes.push(result.outcome);
|
||||||
|
}
|
||||||
|
return outcomes;
|
||||||
|
};
|
||||||
|
const outcomes = (await Promise.all([loop(left), loop(right)])).flat();
|
||||||
|
assert.ok(outcomes.every((outcome) => outcome === "updated"), outcomes.join());
|
||||||
|
assert.equal(stored(path), 100);
|
||||||
|
assert.ok(calls > 100, `the caches raced: apply ran ${calls} times`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a lost attempt reruns apply on the fresh value", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const seen: Array<number | undefined> = [];
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
seen.push(current?.value);
|
||||||
|
if (seen.length === 1) await rival.put(counterDocument, KEY, { value: 100 });
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [1, 100]);
|
||||||
|
assert.equal(result.outcome, "updated");
|
||||||
|
assert.equal(result.document?.value, 101);
|
||||||
|
assert.equal(stored(path), 101);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("exhausted attempts return lostRace, and the cache then shows the store", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
let calls = 0;
|
||||||
|
const result = await cache.update(
|
||||||
|
counterDocument,
|
||||||
|
KEY,
|
||||||
|
async (current) => {
|
||||||
|
calls++;
|
||||||
|
await rival.put(counterDocument, KEY, { value: 100 * calls });
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
},
|
||||||
|
{ maxAttempts: 3 },
|
||||||
|
);
|
||||||
|
assert.equal(calls, 3);
|
||||||
|
assert.equal(result.outcome, "lostRace");
|
||||||
|
assert.equal(result.document?.value, 300);
|
||||||
|
assert.equal(stored(path), 300);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("five attempts by default", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
let calls = 0;
|
||||||
|
const result = await cache.update(counterDocument, KEY, async () => {
|
||||||
|
calls++;
|
||||||
|
await rival.put(counterDocument, KEY, { value: calls });
|
||||||
|
return { value: -1 };
|
||||||
|
});
|
||||||
|
assert.equal(result.outcome, "lostRace");
|
||||||
|
assert.equal(calls, 5);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("maxAttempts 1 makes one call", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
let calls = 0;
|
||||||
|
const apply = async (current: { value: number } | undefined) => {
|
||||||
|
calls++;
|
||||||
|
await rival.put(counterDocument, KEY, { value: 50 + calls });
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
};
|
||||||
|
assert.equal((await cache.update(counterDocument, KEY, apply, { maxAttempts: 1 })).outcome, "lostRace");
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// An apply that declines resolves at once whatever the bound, so a validation that lets a bad value through fails the assertion
|
||||||
|
// below instead of looping: no apply here ever loses a race.
|
||||||
|
test("invalid maxAttempts (undefined is the default, null is refused) and a non-function reject before any call", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
let calls = 0;
|
||||||
|
const apply = () => (calls++, undefined);
|
||||||
|
for (const maxAttempts of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, null as never]) {
|
||||||
|
await assert.rejects(() => cache.update(counterDocument, KEY, apply, { maxAttempts }), RangeError, `maxAttempts ${String(maxAttempts)}`);
|
||||||
|
}
|
||||||
|
await assert.rejects(() => cache.update(counterDocument, KEY, "no" as never), TypeError);
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
assert.equal((await cache.update(counterDocument, KEY, apply, { maxAttempts: undefined })).outcome, "declined");
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("null from apply declines as undefined does: nothing is written", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const before = await readFile(path);
|
||||||
|
const result = await cache.update(counterDocument, KEY, () => null as never);
|
||||||
|
assert.deepEqual(result, { outcome: "declined", document: cache.get(counterDocument, KEY) });
|
||||||
|
assert.deepEqual([...(await readFile(path))], [...before]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a cache with no store refuses a key held as another type before apply runs", async () => {
|
||||||
|
const cache = CultCache.builder().withRegistry(registry).build();
|
||||||
|
await cache.put(tallyDocument, KEY, { count: 1 });
|
||||||
|
let calls = 0;
|
||||||
|
await assert.rejects(() => cache.update(counterDocument, KEY, () => (calls++, { value: 1 })), /is not a "update.counter"/);
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a store attached while apply awaits turns the commit into a lost race, never a blind push", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const writer = await open(path);
|
||||||
|
await writer.put(counterDocument, KEY, { value: 50 });
|
||||||
|
const cache = CultCache.builder().withRegistry(registry).build();
|
||||||
|
const seen: Array<number | undefined> = [];
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
seen.push(current?.value);
|
||||||
|
if (seen.length === 1) cache.addBackingStore(new SingleFileMessagePackBackingStore(path));
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [undefined, 50], "the second attempt reads the store's record");
|
||||||
|
assert.equal(result.outcome, "updated");
|
||||||
|
assert.equal(stored(path), 51);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("put installs the storedAt the store stamped, so the cache and the file agree", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
const [envelope] = readSingleFileStore(path).envelopes;
|
||||||
|
await store.push({ ...envelope!, payload: documentPayloadFromBytes(encode({ value: 99 })), storedAt: "2099-01-01T00:00:00.000Z" });
|
||||||
|
await cache.put(counterDocument, KEY, { value: 2 });
|
||||||
|
const onDisk = readSingleFileStore(path).envelopes[0]!.storedAt;
|
||||||
|
assert.equal(onDisk, "2099-01-01T00:00:00.001Z");
|
||||||
|
assert.equal(cache.getEnvelope(counterDocument, KEY)!.storedAt, onDisk);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("apply gets a fresh copy: changing it changes nothing held, and a lost attempt's change is not seen by the next", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const argument: Array<{ value: number } | undefined> = [];
|
||||||
|
const seen: number[] = [];
|
||||||
|
await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
argument.push(current);
|
||||||
|
assert.notEqual(current, cache.get(counterDocument, KEY));
|
||||||
|
seen.push(current!.value);
|
||||||
|
current!.value += 1000;
|
||||||
|
assert.equal(cache.get(counterDocument, KEY)!.value, 1);
|
||||||
|
if (seen.length === 1) await rival.put(counterDocument, KEY, { value: 7 });
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [1, 7], "the second attempt does not see the first one's change");
|
||||||
|
assert.notEqual(argument[0], argument[1]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an in-place change to the held value is neither applied nor persisted", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, fresh] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
cache.get(counterDocument, KEY)!.value = 999;
|
||||||
|
const seen: number[] = [];
|
||||||
|
const result = await cache.update(counterDocument, KEY, (current) => {
|
||||||
|
seen.push(current!.value);
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [1]);
|
||||||
|
assert.equal(stored(path), 2);
|
||||||
|
await fresh.pullAllBackingStores();
|
||||||
|
assert.equal(fresh.get(counterDocument, KEY)!.value, 2);
|
||||||
|
result.document!.value = 999;
|
||||||
|
await cache.update(counterDocument, KEY, plus(1));
|
||||||
|
assert.equal(stored(path), 3);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("undefined from apply declines: nothing is written", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const before = await readFile(path);
|
||||||
|
const held = await cache.update(counterDocument, KEY, () => undefined);
|
||||||
|
assert.equal(held.outcome, "declined");
|
||||||
|
assert.equal(held.document, cache.get(counterDocument, KEY));
|
||||||
|
const absent = await cache.update(counterDocument, "absent", () => undefined);
|
||||||
|
assert.deepEqual(absent, { outcome: "declined", document: undefined });
|
||||||
|
assert.deepEqual([...(await readFile(path))], [...before]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an absent key is created once under a race", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [left, right] = [await open(path), await open(path)];
|
||||||
|
const outcomes = await Promise.all(
|
||||||
|
[left, right].map((cache) =>
|
||||||
|
cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
await tick();
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.deepEqual(outcomes.map((result) => result.outcome), ["updated", "updated"]);
|
||||||
|
assert.equal(stored(path), 2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a rival writing and restoring the value still reruns apply", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const seen: number[] = [];
|
||||||
|
await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
seen.push(current!.value);
|
||||||
|
if (seen.length === 1) {
|
||||||
|
await rival.put(counterDocument, KEY, { value: 5 });
|
||||||
|
await rival.put(counterDocument, KEY, { value: 1 });
|
||||||
|
}
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [1, 1]);
|
||||||
|
assert.equal(stored(path), 2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a rival removal is seen as undefined", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [cache, rival] = [await open(path), await open(path)];
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const seen: Array<number | undefined> = [];
|
||||||
|
await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
seen.push(current?.value);
|
||||||
|
if (seen.length === 1) {
|
||||||
|
await rival.pullAllBackingStores();
|
||||||
|
await rival.delete(counterDocument, KEY);
|
||||||
|
}
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [1, undefined]);
|
||||||
|
assert.equal(stored(path), 1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a stale cache's first attempt reads the store", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const [stale, rival] = [await open(path), await open(path)];
|
||||||
|
await rival.put(counterDocument, KEY, { value: 1 });
|
||||||
|
await stale.pullAllBackingStores();
|
||||||
|
await rival.put(counterDocument, KEY, { value: 7 });
|
||||||
|
const seen: number[] = [];
|
||||||
|
await stale.update(counterDocument, KEY, (current) => {
|
||||||
|
seen.push(current!.value);
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
});
|
||||||
|
assert.deepEqual(seen, [7]);
|
||||||
|
assert.equal(stored(path), 8);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("refusals are not retried and apply is not called for a record it cannot copy", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(tallyDocument, "tally", { count: 1 });
|
||||||
|
const before = await readFile(path);
|
||||||
|
let calls = 0;
|
||||||
|
const apply = (current: { value: number } | undefined) => {
|
||||||
|
calls++;
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
};
|
||||||
|
// Another type at the key.
|
||||||
|
await assert.rejects(() => cache.update(counterDocument, "tally", apply), /not a "update.counter"/);
|
||||||
|
// A foreign record: a type this cache has no definition for.
|
||||||
|
const narrow = await open(path, defineDocumentRegistry(counterDocument));
|
||||||
|
await assert.rejects(() => narrow.update(counterDocument, "tally", apply), /not a "update.counter"/);
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
assert.deepEqual([...(await readFile(path))], [...before]);
|
||||||
|
|
||||||
|
// A store without read and commitIf cannot take a conditional write.
|
||||||
|
const bare: CacheBackingStore = { pullAll: async () => [], push: async () => {}, delete: async () => {} };
|
||||||
|
const plain = CultCache.builder().withRegistry(registry).withGenericStore(bare).build();
|
||||||
|
await plain.pullAllBackingStores();
|
||||||
|
await assert.rejects(() => plain.update(counterDocument, KEY, apply), /read and commitIf/);
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
|
||||||
|
// A record held read-only is refused by the judge after apply ran once; apply that throws is not retried either.
|
||||||
|
const surface = surfaceDocument();
|
||||||
|
const entries = [envelopeOf(await read("extra-slot-record.msgpack"))];
|
||||||
|
const readOnly = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(entries)).build();
|
||||||
|
await readOnly.pullAllBackingStores();
|
||||||
|
let surfaceCalls = 0;
|
||||||
|
await assert.rejects(() => readOnly.update(surface, "surface", (value) => (surfaceCalls++, value)), SchemaConflictError);
|
||||||
|
assert.equal(surfaceCalls, 1);
|
||||||
|
assert.equal(entries.length, 1, "nothing reached the store");
|
||||||
|
await assert.rejects(
|
||||||
|
() => cache.update(counterDocument, KEY, () => { calls++; throw new Error("boom"); }),
|
||||||
|
/boom/,
|
||||||
|
);
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("own writes cause no retry, and updated returns the held instance", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
let calls = 0;
|
||||||
|
const apply = (current: { value: number } | undefined) => {
|
||||||
|
calls++;
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
};
|
||||||
|
await cache.update(counterDocument, KEY, apply);
|
||||||
|
const second = await cache.update(counterDocument, KEY, apply);
|
||||||
|
assert.equal(calls, 2);
|
||||||
|
assert.equal(second.outcome, "updated");
|
||||||
|
assert.equal(second.document, cache.get(counterDocument, KEY));
|
||||||
|
assert.equal(stored(path), 3);
|
||||||
|
// The cache holds the record as the store wrote it, the stamp the store chose included: over a record stamped in the future
|
||||||
|
// the store writes one millisecond after it, whatever the clock says.
|
||||||
|
const future = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
await new SingleFileMessagePackBackingStore(path).push({ ...future, payload: documentPayloadFromBytes(encode({ value: 99 })), storedAt: "2099-01-01T00:00:00.000Z" });
|
||||||
|
await cache.update(counterDocument, KEY, apply);
|
||||||
|
assert.equal(readSingleFileStore(path).envelopes[0]!.storedAt, "2099-01-01T00:00:00.001Z");
|
||||||
|
assert.equal(cache.getEnvelope(counterDocument, KEY)!.storedAt, "2099-01-01T00:00:00.001Z");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("apply runs outside the cache's serial queue", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
let calls = 0;
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
calls++;
|
||||||
|
await Promise.race([
|
||||||
|
cache.put(tallyDocument, "other", { count: 1 }),
|
||||||
|
new Promise((_, reject) => setTimeout(() => reject(new Error("a cache call from apply did not finish")), 5000).unref()),
|
||||||
|
]);
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
});
|
||||||
|
assert.equal(result.outcome, "updated");
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a cache with no store loses no increment under interleaved updates", async () => {
|
||||||
|
const cache = CultCache.builder().withRegistry(registry).build();
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
let calls = 0;
|
||||||
|
const loop = async () => {
|
||||||
|
let updated = 0;
|
||||||
|
for (let i = 0; i < 50; i++) {
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
calls++;
|
||||||
|
await tick();
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
});
|
||||||
|
if (result.outcome === "updated") updated++;
|
||||||
|
}
|
||||||
|
return updated;
|
||||||
|
};
|
||||||
|
// The two loops run in lockstep, so one may lose every round (a lostRace is the answer, not a lost write): what is held is
|
||||||
|
// exactly what the updated outcomes wrote.
|
||||||
|
const updated = (await Promise.all([loop(), loop()])).reduce((sum, n) => sum + n, 0);
|
||||||
|
assert.equal(cache.get(counterDocument, KEY)!.value, updated);
|
||||||
|
assert.ok(calls > updated, `the loops raced: apply ran ${calls} times for ${updated} writes`);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a store-less cache copies the bytes it admitted, not the held value", async () => {
|
||||||
|
const cache = CultCache.builder().withRegistry(registry).build();
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
cache.get(counterDocument, KEY)!.value = 999;
|
||||||
|
const seen: number[] = [];
|
||||||
|
await cache.update(counterDocument, KEY, (current) => (seen.push(current!.value), { value: current!.value + 1 }));
|
||||||
|
assert.deepEqual(seen, [1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("updateGlobal updates the global record and refuses one loaded from a legacy key until putGlobal retires it", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const settings = defineDocumentRegistry(settingsDocument);
|
||||||
|
const cache = await open(path, settings);
|
||||||
|
const first = await cache.updateGlobal(settingsDocument, (current) => ({ theme: `${current?.theme ?? ""}a` }));
|
||||||
|
assert.equal(first.outcome, "updated");
|
||||||
|
assert.deepEqual(cache.getGlobal(settingsDocument), { theme: "a" });
|
||||||
|
assert.equal(readSingleFileStore(path).envelopes[0]!.key, CultCache.GLOBAL_KEY);
|
||||||
|
await rm(path);
|
||||||
|
|
||||||
|
const legacy = await open(path, defineDocumentRegistry(plainSettingsDocument));
|
||||||
|
await legacy.put(plainSettingsDocument, "old-settings", { theme: "x" });
|
||||||
|
const loaded = await open(path, settings);
|
||||||
|
let calls = 0;
|
||||||
|
await assert.rejects(() => loaded.updateGlobal(settingsDocument, (current) => (calls++, current)), /putGlobal to retire its legacy key/);
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
// Every path to the write is refused, the global's own key included, and the legacy record survives.
|
||||||
|
const before = await readFile(path);
|
||||||
|
await assert.rejects(() => loaded.update(settingsDocument, CultCache.GLOBAL_KEY, () => ({ theme: "lost" })), /putGlobal to retire its legacy key/);
|
||||||
|
assert.deepEqual([...(await readFile(path))], [...before]);
|
||||||
|
assert.deepEqual(readSingleFileStore(path).envelopes.map((e) => e.key), ["old-settings"]);
|
||||||
|
await loaded.putGlobal(settingsDocument, { theme: "y" });
|
||||||
|
assert.equal((await loaded.updateGlobal(settingsDocument, (current) => ({ theme: `${current!.theme}z` }))).outcome, "updated");
|
||||||
|
assert.deepEqual(loaded.getGlobal(settingsDocument), { theme: "yz" });
|
||||||
|
await assert.rejects(() => cache.update(settingsDocument, "not-global", () => ({ theme: "q" })), /must use key/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a legacy global that a pull loads while apply awaits is refused at the write, and its legacy record survives", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const settings = defineDocumentRegistry(settingsDocument);
|
||||||
|
const cache = await open(path, settings);
|
||||||
|
const legacy = await open(path, defineDocumentRegistry(plainSettingsDocument));
|
||||||
|
await assert.rejects(
|
||||||
|
() =>
|
||||||
|
cache.update(settingsDocument, CultCache.GLOBAL_KEY, async () => {
|
||||||
|
await legacy.put(plainSettingsDocument, "old-settings", { theme: "x" });
|
||||||
|
await cache.pullAllBackingStores();
|
||||||
|
return { theme: "lost" };
|
||||||
|
}),
|
||||||
|
/putGlobal to retire its legacy key/,
|
||||||
|
);
|
||||||
|
assert.deepEqual(readSingleFileStore(path).envelopes.map((e) => e.key), ["old-settings"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the ledger holds under contention: the stored value is the number of updated outcomes", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const caches = await Promise.all([open(path), open(path), open(path)]);
|
||||||
|
const results = await Promise.all(
|
||||||
|
caches.map(async (cache) => {
|
||||||
|
let updated = 0;
|
||||||
|
for (let i = 0; i < 25; i++) {
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
await tick();
|
||||||
|
return { value: (current?.value ?? 0) + 1 };
|
||||||
|
});
|
||||||
|
if (result.outcome === "updated") updated++;
|
||||||
|
}
|
||||||
|
return updated;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(stored(path), results.reduce((sum, n) => sum + n, 0));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a flush of an unchanged cache rewrites the file byte for byte, and an update does not lose a race to it", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
await cache.put(counterDocument, "other", { value: 7 });
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
const before = await readFile(path);
|
||||||
|
// What CultMesh.flush does: push the whole stored image, twice.
|
||||||
|
await store.pushAll(await store.pullAll());
|
||||||
|
await store.pushAll(await store.pullAll());
|
||||||
|
assert.deepEqual(await readFile(path), before);
|
||||||
|
const result = await cache.update(counterDocument, KEY, async (current) => {
|
||||||
|
await store.pushAll(await store.pullAll());
|
||||||
|
return { value: current!.value + 1 };
|
||||||
|
}, { maxAttempts: 1 });
|
||||||
|
assert.equal(result.outcome, "updated");
|
||||||
|
assert.equal(stored(path), 2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a changed record is re-stamped later than the one it replaces, and a revert to earlier bytes too", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const store = new SingleFileMessagePackBackingStore(path);
|
||||||
|
const cache = await open(path);
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const first = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
const same = await store.push({ ...first, storedAt: "2099-01-01T00:00:00.000Z" });
|
||||||
|
assert.equal(same.storedAt, first.storedAt, "identical content keeps its stamp");
|
||||||
|
await cache.put(counterDocument, KEY, { value: 2 });
|
||||||
|
const second = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
const reverted = await store.push({ ...first, storedAt: second.storedAt });
|
||||||
|
assert.ok(Date.parse(reverted.storedAt) > Date.parse(second.storedAt));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a push result that is not a parseable stamp is ignored", async () => {
|
||||||
|
for (const result of [true, {}, "", "garbage", { storedAt: "" }, { storedAt: "garbage" }]) {
|
||||||
|
const entries: Parameters<typeof memoryStore>[0] = [];
|
||||||
|
const store = { ...memoryStore(entries), push: async (entry: (typeof entries)[number]) => { entries.push(entry); return result as never; } };
|
||||||
|
const cache = CultCache.builder().withRegistry(registry).withGenericStore(store).build();
|
||||||
|
await cache.put(counterDocument, KEY, { value: 1 });
|
||||||
|
const held = cache.getEnvelope(counterDocument, KEY)!.storedAt;
|
||||||
|
assert.equal(Number.isNaN(Date.parse(held)), false, `held stamp ${held} after a push result of ${JSON.stringify(result)}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a rewrite of identical content keeps the stamp but writes the writer's type and catalog entry", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const old = defineDocumentType({ type: "update.rn.old", schemaId: "update.rn.v1", schemaName: "update.rn.old", schemaVersion: "update.rn.v1", schema: z.object({ value: z.number().int() }) });
|
||||||
|
const renamed = defineDocumentType({ type: "update.rn.new", schemaId: "update.rn.v1", schemaName: "update.rn.new", schemaVersion: "update.rn.v1", schema: z.object({ value: z.number().int() }) });
|
||||||
|
await (await open(path, defineDocumentRegistry(old))).put(old, KEY, { value: 1 });
|
||||||
|
const before = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
await (await open(path, defineDocumentRegistry(renamed))).put(renamed, KEY, { value: 1 });
|
||||||
|
const after = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
assert.equal(after.storedAt, before.storedAt, "identical content keeps its stamp");
|
||||||
|
assert.equal(after.type, "update.rn.new", "the writer's type is what the file says");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a rewrite of the same bytes under another schema id is a changed record: stamped later, written under the new id", async () => {
|
||||||
|
await withDirectory(async (path) => {
|
||||||
|
const v1 = defineDocumentType({ type: "update.sid", schemaId: "update.sid.v1", schemaName: "update.sid", schemaVersion: "update.sid.v1", schema: z.object({ value: z.number().int() }) });
|
||||||
|
const v2 = defineDocumentType({ type: "update.sid", schemaId: "update.sid.v2", schemaName: "update.sid", schemaVersion: "update.sid.v2", compatibleSchemaIds: ["update.sid.v1"], schema: z.object({ value: z.number().int() }) });
|
||||||
|
await (await open(path, defineDocumentRegistry(v1))).put(v1, KEY, { value: 1 });
|
||||||
|
const before = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
await (await open(path, defineDocumentRegistry(v2))).put(v2, KEY, { value: 1 });
|
||||||
|
const after = readSingleFileStore(path).envelopes[0]!;
|
||||||
|
assert.ok(Date.parse(after.storedAt) > Date.parse(before.storedAt));
|
||||||
|
assert.equal(schemaIdForEnvelope(after), "update.sid.v2");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -147,8 +147,8 @@ CultMath names with tests instead of copied project-local helper drift.
|
||||||
```powershell
|
```powershell
|
||||||
dotnet build packages/cultmath/src/CultMath/CultMath.csproj
|
dotnet build packages/cultmath/src/CultMath/CultMath.csproj
|
||||||
dotnet test packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj
|
dotnet test packages/cultmath/tests/CultMath.Tests/CultMath.Tests.csproj
|
||||||
cargo test --manifest-path packages/cultmath/native/cultmath-core/Cargo.toml
|
cargo test --locked --manifest-path packages/cultmath/native/cultmath-core/Cargo.toml
|
||||||
cargo build --release --manifest-path packages/cultmath/native/cultmath-core/Cargo.toml
|
cargo build --locked --release --manifest-path packages/cultmath/native/cultmath-core/Cargo.toml
|
||||||
.\packages\cultmath\scripts\build-unity-package.ps1
|
.\packages\cultmath\scripts\build-unity-package.ps1
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@
|
||||||
"node": ">=20"
|
"node": ">=20"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@gamecult/cultcache-ts": "^0.15.0",
|
"@gamecult/cultcache-ts": "^0.16.0",
|
||||||
"@gamecult/cultnet-ts": "^0.4.0"
|
"@gamecult/cultnet-ts": "^0.4.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ from typing import Any, Callable
|
||||||
from cultcache_py import CultCache, CultCacheEnvelope, SingleFileMessagePackBackingStore
|
from cultcache_py import CultCache, CultCacheEnvelope, SingleFileMessagePackBackingStore
|
||||||
from cultcache_py.cache import CultCacheError, index_text
|
from cultcache_py.cache import CultCacheError, index_text
|
||||||
from cultcache_py.documents import DocumentDefinition, extract_value
|
from cultcache_py.documents import DocumentDefinition, extract_value
|
||||||
|
from cultnet_py.replication import raw_record_to_envelope
|
||||||
from cultnet_py import (
|
from cultnet_py import (
|
||||||
CultNetAppliedRecord,
|
CultNetAppliedRecord,
|
||||||
CultNetFileShardMutationLogStore,
|
CultNetFileShardMutationLogStore,
|
||||||
|
|
@ -570,12 +571,14 @@ class CultMeshDatabase:
|
||||||
catalog_entry=catalog_entry,
|
catalog_entry=catalog_entry,
|
||||||
)
|
)
|
||||||
value = self.cache.put_envelope(registered, envelope)
|
value = self.cache.put_envelope(registered, envelope)
|
||||||
|
# The stamp the write was stored at, which a replacing write mints later than the record it replaces: the cache holds it.
|
||||||
|
stored_at = self.cache.get_required_envelope(registered, key).stored_at
|
||||||
self._publish_local_change(registered, key, "added" if previous is None else "updated", value, previous)
|
self._publish_local_change(registered, key, "added" if previous is None else "updated", value, previous)
|
||||||
message = document_put_raw(
|
message = document_put_raw(
|
||||||
message_id=message_id,
|
message_id=message_id,
|
||||||
key=key,
|
key=key,
|
||||||
schema_id=envelope.schema_id or catalog_entry.schema_id,
|
schema_id=envelope.schema_id or catalog_entry.schema_id,
|
||||||
stored_at=envelope.stored_at,
|
stored_at=stored_at,
|
||||||
payload=envelope.payload,
|
payload=envelope.payload,
|
||||||
source_runtime_id=self.runtime_id,
|
source_runtime_id=self.runtime_id,
|
||||||
shard_id=shard_id,
|
shard_id=shard_id,
|
||||||
|
|
@ -662,10 +665,10 @@ class CultMeshDatabase:
|
||||||
):
|
):
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
prepared = self._raw_put_envelope(message)
|
document = self._raw_put_document(message)
|
||||||
except Exception: # noqa: BLE001 - any other malformed shape
|
except Exception: # noqa: BLE001 - any other malformed shape
|
||||||
return None
|
return None
|
||||||
return None if prepared is None else (prepared[0].catalog_entry().schema_id, prepared[1].key)
|
return None if document is None else (document.catalog_entry().schema_id, document_record["recordKey"])
|
||||||
|
|
||||||
def raw_delete_target(self, message: dict[str, Any]) -> tuple[str, str] | None:
|
def raw_delete_target(self, message: dict[str, Any]) -> tuple[str, str] | None:
|
||||||
"""The canonical schema id and record key a raw delete names, or None when its schema is not registered."""
|
"""The canonical schema id and record key a raw delete names, or None when its schema is not registered."""
|
||||||
|
|
@ -683,10 +686,11 @@ class CultMeshDatabase:
|
||||||
value = self.cache.put_envelope(document, envelope)
|
value = self.cache.put_envelope(document, envelope)
|
||||||
change_kind = "added" if previous is None else "updated"
|
change_kind = "added" if previous is None else "updated"
|
||||||
change = self._publish_local_change(document, record_key, change_kind, value, previous)
|
change = self._publish_local_change(document, record_key, change_kind, value, previous)
|
||||||
self._append_shard_log_put(CultNetMessage(
|
# The log carries the stamp the cache holds, which a replacing put mints later than the record it replaces, not the peer's.
|
||||||
"cultnet.document_put_raw.v0",
|
stored_at = self.cache.get_required_envelope(document, record_key).stored_at
|
||||||
{key: value for key, value in message.items() if key != "schemaVersion"},
|
logged = {key: value for key, value in message.items() if key != "schemaVersion"}
|
||||||
), change_kind)
|
logged["document"] = {**message["document"], "storedAt": stored_at}
|
||||||
|
self._append_shard_log_put(CultNetMessage("cultnet.document_put_raw.v0", logged), change_kind)
|
||||||
return change
|
return change
|
||||||
|
|
||||||
def raw_put_served_alone(self, message: dict[str, Any], *, message_id: str) -> dict[str, Any] | None:
|
def raw_put_served_alone(self, message: dict[str, Any], *, message_id: str) -> dict[str, Any] | None:
|
||||||
|
|
@ -703,27 +707,25 @@ class CultMeshDatabase:
|
||||||
documents=(self._served_record(prepared[1]),),
|
documents=(self._served_record(prepared[1]),),
|
||||||
).to_wire()
|
).to_wire()
|
||||||
|
|
||||||
def _raw_put_envelope(
|
def _raw_put_document(self, message: dict[str, Any]) -> DocumentDefinition[Any] | None:
|
||||||
self, message: dict[str, Any]
|
"""The registered document a raw put names, or None when its record is not a map or its schema is not registered."""
|
||||||
) -> tuple[DocumentDefinition[Any], CultCacheEnvelope] | None:
|
|
||||||
document_record = message.get("document")
|
document_record = message.get("document")
|
||||||
if not isinstance(document_record, dict):
|
if not isinstance(document_record, dict):
|
||||||
return None
|
return None
|
||||||
schema_id = str(document_record.get("schemaId") or "")
|
schema_id = str(document_record.get("schemaId") or "")
|
||||||
documents_by_schema_id = schema_document_map(self.documents)
|
|
||||||
try:
|
try:
|
||||||
document = resolve_document_for_raw_record(documents_by_schema_id, schema_id, document_record)
|
return resolve_document_for_raw_record(schema_document_map(self.documents), schema_id, document_record)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return None
|
return None
|
||||||
# One id: the local schema id, under which the cache stores the record.
|
|
||||||
return document, CultCacheEnvelope(
|
def _raw_put_envelope(
|
||||||
key=str(document_record["recordKey"]),
|
self, message: dict[str, Any]
|
||||||
type=document.type,
|
) -> tuple[DocumentDefinition[Any], CultCacheEnvelope] | None:
|
||||||
schema_id=document.catalog_entry().schema_id,
|
document = self._raw_put_document(message)
|
||||||
payload=bytes(document_record["payload"]),
|
if document is None:
|
||||||
stored_at=str(document_record.get("storedAt") or ""),
|
return None
|
||||||
catalog_entry=document.catalog_entry(),
|
# One id: the local schema id, under which the cache stores the record. The stamp is judged where a record becomes an envelope.
|
||||||
)
|
return document, raw_record_to_envelope(document, document.catalog_entry().schema_id, message["document"])
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _served_record(envelope: CultCacheEnvelope) -> CultNetRawDocumentRecord:
|
def _served_record(envelope: CultCacheEnvelope) -> CultNetRawDocumentRecord:
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ from typing import Any
|
||||||
|
|
||||||
import msgpack
|
import msgpack
|
||||||
|
|
||||||
|
from cultcache_py.stores import parse_stored_at
|
||||||
from cultnet_py import (
|
from cultnet_py import (
|
||||||
error_message,
|
error_message,
|
||||||
message_invalid_error,
|
message_invalid_error,
|
||||||
|
|
@ -686,9 +687,11 @@ class CultMeshLocalServer:
|
||||||
message_id=str(message.get("messageId") or ""),
|
message_id=str(message.get("messageId") or ""),
|
||||||
code="unregistered_document_put",
|
code="unregistered_document_put",
|
||||||
)]
|
)]
|
||||||
|
# Subscribers are sent the stamp the store holds, which a replacing put mints later than the record it replaces, not the peer's.
|
||||||
|
stored = self.node.database.cache.get_required_envelope(change.document, change.record_key).stored_at
|
||||||
return self._database_change_notifications(
|
return self._database_change_notifications(
|
||||||
message,
|
message,
|
||||||
document_record,
|
{**document_record, "storedAt": stored},
|
||||||
change.change_kind,
|
change.change_kind,
|
||||||
subscriptions,
|
subscriptions,
|
||||||
resolved_schema_id=change.schema_id,
|
resolved_schema_id=change.schema_id,
|
||||||
|
|
@ -967,7 +970,7 @@ class CultMeshLocalServer:
|
||||||
as a snapshot would serve it under the shortest message id, must fit max_snapshot_bytes;
|
as a snapshot would serve it under the shortest message id, must fit max_snapshot_bytes;
|
||||||
a put arriving over RUDP must also fit, as fragments of rudp_max_fragment_bytes, in one
|
a put arriving over RUDP must also fit, as fragments of rudp_max_fragment_bytes, in one
|
||||||
message and the peer's reliable queue. A put whose document has no string recordKey or
|
message and the peer's reliable queue. A put whose document has no string recordKey or
|
||||||
binary payload could never be stored or served, and is refused as malformed."""
|
binary payload, or a storedAt that is no stamp, could never be stored or served, and is refused as malformed."""
|
||||||
document_record = message.get("document")
|
document_record = message.get("document")
|
||||||
if not (
|
if not (
|
||||||
isinstance(document_record, dict)
|
isinstance(document_record, dict)
|
||||||
|
|
@ -975,6 +978,8 @@ class CultMeshLocalServer:
|
||||||
and isinstance(document_record.get("payload"), (bytes, bytearray))
|
and isinstance(document_record.get("payload"), (bytes, bytearray))
|
||||||
):
|
):
|
||||||
return self._message_invalid(message, "document", "Raw put documents must carry a string recordKey and a binary payload.")
|
return self._message_invalid(message, "document", "Raw put documents must carry a string recordKey and a binary payload.")
|
||||||
|
if parse_stored_at(document_record.get("storedAt")) is None:
|
||||||
|
return self._message_invalid(message, "document", "Raw put documents must carry a storedAt stamp.")
|
||||||
if self.max_snapshot_bytes is None and not over_rudp:
|
if self.max_snapshot_bytes is None and not over_rudp:
|
||||||
return None
|
return None
|
||||||
alone = self.node.database.raw_put_served_alone(message, message_id=_SHORTEST_SERVED_MESSAGE_ID)
|
alone = self.node.database.raw_put_served_alone(message, message_id=_SHORTEST_SERVED_MESSAGE_ID)
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ from typing import Any, Callable
|
||||||
import msgpack
|
import msgpack
|
||||||
from cultcache_py import (
|
from cultcache_py import (
|
||||||
CultCache,
|
CultCache,
|
||||||
|
CultCacheEnvelope,
|
||||||
SingleFileMessagePackBackingStore,
|
SingleFileMessagePackBackingStore,
|
||||||
define_database_entry_type,
|
define_database_entry_type,
|
||||||
define_document_type,
|
define_document_type,
|
||||||
|
|
@ -796,6 +797,51 @@ class CultMeshTests(unittest.TestCase):
|
||||||
self.assertEqual(node.get_required(document, "note:second").body, "second catalog publication")
|
self.assertEqual(node.get_required(document, "note:second").body, "second catalog publication")
|
||||||
self.assertIsInstance(node.get_required(alias, "note:second"), UiNote)
|
self.assertIsInstance(node.get_required(alias, "note:second"), UiNote)
|
||||||
|
|
||||||
|
def test_raw_put_publishes_and_logs_the_stamp_the_store_wrote(self) -> None:
|
||||||
|
document = define_database_entry_type("mesh.stamp_item", [("name", 0), ("category", 1), ("value", 2, 0)], cls=Item)
|
||||||
|
node = create_node(runtime_id="python-node")
|
||||||
|
node.register_document(document)
|
||||||
|
cache = node.database.cache
|
||||||
|
entry = document.catalog_entry()
|
||||||
|
future = "2099-01-01T00:00:00.000000+00:00"
|
||||||
|
cache.put_envelope(
|
||||||
|
document,
|
||||||
|
CultCacheEnvelope(
|
||||||
|
key="item:stamp",
|
||||||
|
type=document.type,
|
||||||
|
payload=document.encode_payload(Item("old", "gear", 1)),
|
||||||
|
stored_at=future,
|
||||||
|
schema_id=entry.schema_id,
|
||||||
|
catalog_entry=entry,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# A replacing write is stored later than the record it replaces, so the stamp is not the one the write was created with.
|
||||||
|
wire = node.put_raw_message(document, "item:stamp", Item("new", "gear", 2), shard_id="mesh", shard_epoch=1).to_wire()
|
||||||
|
held = cache.get_required_envelope(document, "item:stamp").stored_at
|
||||||
|
self.assertGreater(held, future)
|
||||||
|
self.assertEqual(wire["document"]["storedAt"], held)
|
||||||
|
self.assertEqual(node.database._shard_logs["mesh"][-1]["put"]["document"]["storedAt"], held)
|
||||||
|
|
||||||
|
def test_applied_raw_put_logs_the_stamp_the_cache_holds(self) -> None:
|
||||||
|
document = define_database_entry_type("mesh.applied_stamp", [("name", 0), ("category", 1), ("value", 2, 0)], cls=Item)
|
||||||
|
node = create_node(runtime_id="python-node")
|
||||||
|
node.register_document(document)
|
||||||
|
cache = node.database.cache
|
||||||
|
entry = document.catalog_entry()
|
||||||
|
|
||||||
|
def envelope(value: int, stored_at: str) -> CultCacheEnvelope:
|
||||||
|
return CultCacheEnvelope("item:applied", document.type, document.encode_payload(Item("n", "gear", value)), stored_at, entry.schema_id, entry)
|
||||||
|
|
||||||
|
cache.put_envelope(document, envelope(1, "2099-01-01T00:00:00.000000+00:00"))
|
||||||
|
peer = node.put_raw_message(document, "item:peer", Item("peer", "gear", 2)).to_wire() # a well-formed put to replay
|
||||||
|
peer["document"] = {**peer["document"], "recordKey": "item:applied", "storedAt": "2000-01-01T00:00:00Z",
|
||||||
|
"payload": envelope(2, "x").payload}
|
||||||
|
peer["shardId"], peer["shardEpoch"] = "mesh", 1
|
||||||
|
node.database.apply_raw_put_message(peer)
|
||||||
|
held = cache.get_required_envelope(document, "item:applied").stored_at
|
||||||
|
self.assertGreater(held, "2099-01-01T00:00:00.000000+00:00")
|
||||||
|
self.assertEqual(node.database._shard_logs["mesh"][-1]["put"]["document"]["storedAt"], held)
|
||||||
|
|
||||||
def test_cultmesh_node_emits_raw_put_and_delete_messages_for_local_writes(self) -> None:
|
def test_cultmesh_node_emits_raw_put_and_delete_messages_for_local_writes(self) -> None:
|
||||||
document = define_database_entry_type(
|
document = define_database_entry_type(
|
||||||
"mesh.emit_item",
|
"mesh.emit_item",
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ the peer the connection loop would pass, and one test drives the same path over
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
@ -23,7 +24,7 @@ from cultmesh_py import (
|
||||||
CultMeshServerPeer,
|
CultMeshServerPeer,
|
||||||
create_node,
|
create_node,
|
||||||
)
|
)
|
||||||
from cultmesh_py.server import CultMeshLocalServer
|
from cultmesh_py.server import CultMeshLocalServer, _DatabaseSubscription
|
||||||
from cultnet_py import CultNetPeerError, create_rudp_schema_transport, document_delete, document_put_raw, error_message
|
from cultnet_py import CultNetPeerError, create_rudp_schema_transport, document_delete, document_put_raw, error_message
|
||||||
|
|
||||||
SCHEMA_ID = "sha256:authority-note"
|
SCHEMA_ID = "sha256:authority-note"
|
||||||
|
|
@ -55,12 +56,12 @@ class Recording:
|
||||||
return self._decide(request)
|
return self._decide(request)
|
||||||
|
|
||||||
|
|
||||||
def put_of(key: str, text: str, *, schema_id: str = SCHEMA_ID) -> dict[str, Any]:
|
def put_of(key: str, text: str, *, schema_id: str = SCHEMA_ID, stored_at: str = "2026-11-07T00:00:00Z") -> dict[str, Any]:
|
||||||
return document_put_raw(
|
return document_put_raw(
|
||||||
message_id="wire",
|
message_id="wire",
|
||||||
key=key,
|
key=key,
|
||||||
schema_id=schema_id,
|
schema_id=schema_id,
|
||||||
stored_at="2026-11-07T00:00:00Z",
|
stored_at=stored_at,
|
||||||
payload=NOTE.encode_payload({"schema_version": "tests.authority_note.v1", "text": text}),
|
payload=NOTE.encode_payload({"schema_version": "tests.authority_note.v1", "text": text}),
|
||||||
shard_id=SHARD_ID,
|
shard_id=SHARD_ID,
|
||||||
shard_epoch=1,
|
shard_epoch=1,
|
||||||
|
|
@ -73,6 +74,64 @@ def delete_of(key: str, *, schema_id: str = SCHEMA_ID) -> dict[str, Any]:
|
||||||
).to_wire()
|
).to_wire()
|
||||||
|
|
||||||
|
|
||||||
|
class NotificationStampTests(unittest.TestCase):
|
||||||
|
"""A subscriber is sent the stamp the store holds after a peer's put, as the shard log is, not the peer's stamp."""
|
||||||
|
|
||||||
|
def test_the_notification_carries_the_stamp_the_cache_holds(self) -> None:
|
||||||
|
local, peer_stamp = "2099-01-01T00:00:00.000000+00:00", "2000-01-01T00:00:00Z"
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
for backing in (None, Path(directory) / "node.cc"): # store-less, then store-backed
|
||||||
|
with self.subTest(store=backing is not None):
|
||||||
|
node = create_node(backing, runtime_id="stamp-server")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
server = CultMeshLocalServer(node=node, port=0, write_authority=CultMeshGrantAllWrites())
|
||||||
|
node.database.apply_raw_put_message(put_of(KEY_A, "server", stored_at=local))
|
||||||
|
subscriptions = {"s": _DatabaseSubscription("s", set(), set())}
|
||||||
|
replies = server._handle_connection_message(put_of(KEY_A, "peer", stored_at=peer_stamp), subscriptions, peer=PEER)
|
||||||
|
held = node.cache.get_required_envelope(NOTE, KEY_A).stored_at
|
||||||
|
self.assertGreater(held, local)
|
||||||
|
self.assertEqual([reply["document"]["storedAt"] for reply in replies], [held])
|
||||||
|
self.assertEqual(node.database._shard_logs[SHARD_ID][-1]["put"]["document"]["storedAt"], held)
|
||||||
|
|
||||||
|
|
||||||
|
class StamplessPutTests(unittest.TestCase):
|
||||||
|
"""A peer's put with no storedAt, or one that is no stamp, is refused message_invalid; nothing is minted or stored."""
|
||||||
|
|
||||||
|
def test_a_put_without_a_stamp_is_refused_and_the_store_stays_writable(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = create_node(Path(directory) / "node.cc", runtime_id="stampless-server")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
server = CultMeshLocalServer(node=node, port=0, write_authority=CultMeshGrantAllWrites())
|
||||||
|
sized = CultMeshLocalServer(node=node, port=0, write_authority=CultMeshGrantAllWrites(), max_snapshot_bytes=10**6)
|
||||||
|
rudp = CultMeshLocalServer(node=node, port=0, write_authority=CultMeshGrantAllWrites())
|
||||||
|
for label, stamp in (("missing", None), ("empty", ""), ("garbage", "yesterday"), ("no zone", "2026-11-07T00:00:00")):
|
||||||
|
with self.subTest(label):
|
||||||
|
message = put_of(KEY_A, "peer")
|
||||||
|
del message["document"]["storedAt"]
|
||||||
|
if stamp is not None:
|
||||||
|
message["document"]["storedAt"] = stamp
|
||||||
|
# The size-limited and RUDP paths measure the document as a snapshot would serve it, so they must refuse first.
|
||||||
|
for limited in (server, sized, rudp):
|
||||||
|
[refusal] = limited._handle_connection_message(message, {}, over_rudp=limited is rudp, peer=PEER)
|
||||||
|
self.assertEqual(refusal["code"], "message_invalid")
|
||||||
|
self.assertEqual(refusal["details"]["field"], "document")
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
node.database.apply_raw_put_message(message)
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, KEY_A))
|
||||||
|
self.assertNotIn(SHARD_ID, node.database._shard_logs)
|
||||||
|
# An aliased put with a bad stamp still asks the authority about the canonical id, and is refused before the stamp is judged.
|
||||||
|
authority = Recording(lambda request: CultMeshRemoteWriteDecision.refuse("no"))
|
||||||
|
guarded = CultMeshLocalServer(node=node, port=0, write_authority=authority)
|
||||||
|
aliased = put_of(KEY_A, "peer", schema_id=ALIAS, stored_at="yesterday")
|
||||||
|
[refusal] = guarded._handle_connection_message(aliased, {}, peer=PEER)
|
||||||
|
self.assertEqual(refusal["code"], "write_not_authorized")
|
||||||
|
self.assertEqual(authority.requests[0].schema_id, SCHEMA_ID)
|
||||||
|
# The refused puts left the file readable: the next put of any key is stored.
|
||||||
|
replies = server._handle_connection_message(put_of(KEY_B, "stamped"), {}, peer=PEER)
|
||||||
|
self.assertEqual(replies, [])
|
||||||
|
self.assertEqual(node.cache.get(NOTE, KEY_B)["text"], "stamped")
|
||||||
|
|
||||||
|
|
||||||
class RemoteWriteAuthorityTests(unittest.TestCase):
|
class RemoteWriteAuthorityTests(unittest.TestCase):
|
||||||
def build(self, authority: Any = None) -> CultMeshLocalServer:
|
def build(self, authority: Any = None) -> CultMeshLocalServer:
|
||||||
node = create_node(runtime_id="authority-server")
|
node = create_node(runtime_id="authority-server")
|
||||||
|
|
|
||||||
212
packages/cultmesh-py/tests/test_stampless_replication.py
Normal file
212
packages/cultmesh-py/tests/test_stampless_replication.py
Normal file
|
|
@ -0,0 +1,212 @@
|
||||||
|
"""A record a peer replicates to this node is stored under the stamp it carries, so one with no stamp is refused where it becomes an envelope.
|
||||||
|
|
||||||
|
The stamp shape is tests/vectors/stored-at-stamp/stamps.msgpack, held to the same answers as the raw put of the server
|
||||||
|
(variants:ruling:raw-put-without-stamp-refused). A snapshot response is refused whole with ValueError before anything is stored (as a record of an unregistered schema is refused whole). A shard log is
|
||||||
|
refused whole too: every put it would apply is judged for its stamp before the first entry is stored (a put of an unregistered schema is skipped, as ever, and is not judged). The refusal of a put is
|
||||||
|
message_invalid (field `document`), tested in test_remote_write_authority.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
from cultcache_py import define_database_entry_type
|
||||||
|
from cultcache_py.stores import parse_stored_at
|
||||||
|
from cultmesh_py import CultMeshGrantAllWrites, CultMeshServerPeer, create_node
|
||||||
|
from cultmesh_py.client import CultMeshDocumentSubscription
|
||||||
|
from cultmesh_py.server import CultMeshLocalServer
|
||||||
|
from cultnet_py import CultNetDatabaseChange, document_put_raw
|
||||||
|
|
||||||
|
SCHEMA_ID = "sha256:stampless-note"
|
||||||
|
SHARD_ID = "stampless-shard"
|
||||||
|
PEER = CultMeshServerPeer("127.0.0.1:50000", "tcp")
|
||||||
|
VECTOR = Path(__file__).resolve().parents[3] / "tests/vectors/stored-at-stamp/stamps.msgpack"
|
||||||
|
ROWS: list[tuple[str, bool]] = [(text, parses) for text, parses in msgpack.unpackb(VECTOR.read_bytes(), raw=False)]
|
||||||
|
# The vector's texts, plus the shapes a record can lack a stamp in: the field missing, and not a string at all.
|
||||||
|
REFUSED: list[tuple[str, Any]] = [(f"vector {text!r}", text) for text, parses in ROWS if not parses] + [
|
||||||
|
("missing", None),
|
||||||
|
("not a string", 12345),
|
||||||
|
]
|
||||||
|
|
||||||
|
NOTE = define_database_entry_type(
|
||||||
|
"tests.stampless_note",
|
||||||
|
[("schema_version", 0), ("text", 1)],
|
||||||
|
schema_id=SCHEMA_ID,
|
||||||
|
schema_name="tests.stampless_note",
|
||||||
|
schema_version="tests.stampless_note.v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def record_of(key: str, text: str, stamp: Any = "2026-11-07T00:00:00Z") -> dict[str, Any]:
|
||||||
|
record = {
|
||||||
|
"schemaId": SCHEMA_ID,
|
||||||
|
"recordKey": key,
|
||||||
|
"payloadEncoding": "messagepack",
|
||||||
|
"payload": NOTE.encode_payload({"schema_version": "tests.stampless_note.v1", "text": text}),
|
||||||
|
}
|
||||||
|
if stamp is not None:
|
||||||
|
record["storedAt"] = stamp
|
||||||
|
return record
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_of(*records: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": "snapshot", "documents": list(records)}
|
||||||
|
|
||||||
|
|
||||||
|
def shard_log_of(*records: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
entries = [
|
||||||
|
{
|
||||||
|
"sequence": index + 1,
|
||||||
|
"committedAt": "2026-11-07T00:00:00Z",
|
||||||
|
"changeKind": "added",
|
||||||
|
"put": document_put_raw(
|
||||||
|
message_id=f"put-{index}", key=record["recordKey"], schema_id=SCHEMA_ID, stored_at="2026-11-07T00:00:00Z",
|
||||||
|
payload=record["payload"], shard_id=SHARD_ID, shard_epoch=1,
|
||||||
|
).to_wire() | {"document": record},
|
||||||
|
}
|
||||||
|
for index, record in enumerate(records)
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"schemaVersion": "cultnet.shard_log_response.v0", "messageId": "log", "shardId": SHARD_ID, "shardEpoch": 1,
|
||||||
|
"entries": entries, "resyncRequired": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class StampVectorTests(unittest.TestCase):
|
||||||
|
def test_python_parses_every_row_of_the_vector_as_the_vector_says(self) -> None:
|
||||||
|
self.assertGreater(len(ROWS), 30)
|
||||||
|
for text, parses in ROWS:
|
||||||
|
with self.subTest(text=text):
|
||||||
|
self.assertEqual(parse_stored_at(text) is not None, parses)
|
||||||
|
|
||||||
|
def test_every_vector_row_through_the_server_put_handler_is_stored_or_refused_as_the_vector_says(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = create_node(Path(directory) / "node.cc", runtime_id="vector-server")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
server = CultMeshLocalServer(node=node, port=0, write_authority=CultMeshGrantAllWrites())
|
||||||
|
sized = CultMeshLocalServer(node=node, port=0, write_authority=CultMeshGrantAllWrites(), max_snapshot_bytes=10**6)
|
||||||
|
for index, (text, parses) in enumerate(ROWS):
|
||||||
|
for label, handler, over_rudp in (("plain", server, False), ("sized", sized, False), ("rudp", server, True)):
|
||||||
|
with self.subTest(text=text, path=label):
|
||||||
|
key = f"vector:{index}:{label}"
|
||||||
|
message = document_put_raw(
|
||||||
|
message_id="wire", key=key, schema_id=SCHEMA_ID, stored_at=text,
|
||||||
|
payload=record_of(key, "peer")["payload"], shard_id=SHARD_ID, shard_epoch=1,
|
||||||
|
).to_wire()
|
||||||
|
replies = handler._handle_connection_message(message, {}, over_rudp=over_rudp, peer=PEER)
|
||||||
|
if parses:
|
||||||
|
self.assertEqual(replies, [])
|
||||||
|
self.assertEqual(node.cache.get(NOTE, key)["text"], "peer")
|
||||||
|
else:
|
||||||
|
self.assertEqual([(r["code"], r["details"]["field"]) for r in replies], [("message_invalid", "document")])
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, key))
|
||||||
|
|
||||||
|
|
||||||
|
class StamplessSnapshotTests(unittest.TestCase):
|
||||||
|
def build(self, directory: str) -> Any:
|
||||||
|
node = create_node(Path(directory) / "node.cc", runtime_id="stampless-client")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
node.database.pull()
|
||||||
|
return node
|
||||||
|
|
||||||
|
def test_a_snapshot_with_a_stampless_record_is_refused_whole_and_the_store_survives_a_reopen(self) -> None:
|
||||||
|
for label, stamp in REFUSED:
|
||||||
|
with self.subTest(label), tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = self.build(directory)
|
||||||
|
good, bad = record_of("note:good", "good"), record_of("note:bad", "bad", stamp)
|
||||||
|
for order in ((good, bad), (bad, good)):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
node.database.apply_snapshot_response(snapshot_of(*order))
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, "note:good"), "a refused snapshot stores none of its records")
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, "note:bad"))
|
||||||
|
# The store file reads back, and is still writable by a snapshot of stamped records.
|
||||||
|
reopened = self.build(directory)
|
||||||
|
self.assertIsNone(reopened.cache.get(NOTE, "note:bad"))
|
||||||
|
applied = reopened.database.apply_snapshot_response(snapshot_of(good))
|
||||||
|
self.assertEqual([a.record_key for a in applied], ["note:good"])
|
||||||
|
self.assertEqual(self.build(directory).cache.get(NOTE, "note:good")["text"], "good")
|
||||||
|
|
||||||
|
def test_a_snapshot_of_every_stamped_vector_row_is_stored(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = self.build(directory)
|
||||||
|
rows = [(index, text) for index, (text, parses) in enumerate(ROWS) if parses]
|
||||||
|
node.database.apply_snapshot_response(snapshot_of(*[record_of(f"stamped:{i}", "ok", text) for i, text in rows]))
|
||||||
|
self.assertEqual(len(self.build(directory).cache.get_all(NOTE)), len(rows))
|
||||||
|
for index, _ in rows:
|
||||||
|
self.assertEqual(node.cache.get(NOTE, f"stamped:{index}")["text"], "ok")
|
||||||
|
|
||||||
|
|
||||||
|
class StamplessShardLogTests(unittest.TestCase):
|
||||||
|
def test_a_shard_log_put_with_no_stamp_is_refused_and_the_store_stays_writable(self) -> None:
|
||||||
|
for label, stamp in REFUSED:
|
||||||
|
with self.subTest(label), tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = create_node(Path(directory) / "node.cc", runtime_id="stampless-log")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
node.database.apply_shard_log_response(shard_log_of(record_of("note:bad", "bad", stamp)))
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, "note:bad"))
|
||||||
|
reopened = create_node(Path(directory) / "node.cc", runtime_id="stampless-log")
|
||||||
|
reopened.database.register_document(NOTE)
|
||||||
|
reopened.database.pull()
|
||||||
|
reopened.database.apply_shard_log_response(shard_log_of(record_of("note:good", "good")))
|
||||||
|
self.assertEqual(reopened.cache.get(NOTE, "note:good")["text"], "good")
|
||||||
|
again = create_node(Path(directory) / "node.cc", runtime_id="stampless-log")
|
||||||
|
again.database.register_document(NOTE)
|
||||||
|
again.database.pull()
|
||||||
|
self.assertEqual(again.cache.get(NOTE, "note:good")["text"], "good")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_shard_log_with_a_bad_entry_in_the_middle_stores_nothing_and_tells_no_watcher(self) -> None:
|
||||||
|
for label, stamp in REFUSED:
|
||||||
|
with self.subTest(label), tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = create_node(Path(directory) / "node.cc", runtime_id="stampless-multi")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
seen: list[Any] = []
|
||||||
|
node.database.watch(seen.append)
|
||||||
|
log = shard_log_of(record_of("note:a", "a"), record_of("note:b", "b", stamp), record_of("note:c", "c"))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
node.database.apply_shard_log_response(log)
|
||||||
|
for key in ("note:a", "note:b", "note:c"):
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, key), f"{key} was not stored")
|
||||||
|
self.assertEqual(seen, [], "no watcher heard of a refused log")
|
||||||
|
node.database.apply_shard_log_response(shard_log_of(record_of("note:a", "a"), record_of("note:c", "c")))
|
||||||
|
self.assertEqual([node.cache.get(NOTE, k)["text"] for k in ("note:a", "note:c")], ["a", "c"])
|
||||||
|
self.assertEqual(len(seen), 2, "a later good log applies and is heard")
|
||||||
|
|
||||||
|
|
||||||
|
class _StubSubscription:
|
||||||
|
def __init__(self, change: CultNetDatabaseChange) -> None:
|
||||||
|
self._change = change
|
||||||
|
|
||||||
|
def read_next_change(self) -> CultNetDatabaseChange:
|
||||||
|
return self._change
|
||||||
|
|
||||||
|
|
||||||
|
class StamplessClientReplayTests(unittest.TestCase):
|
||||||
|
def test_a_subscribed_put_with_no_stamp_is_refused_by_read_next_change_and_nothing_is_stored(self) -> None:
|
||||||
|
for label, stamp in REFUSED:
|
||||||
|
with self.subTest(label), tempfile.TemporaryDirectory() as directory:
|
||||||
|
node = create_node(Path(directory) / "node.cc", runtime_id="stampless-subscriber")
|
||||||
|
node.database.register_document(NOTE)
|
||||||
|
subscription = CultMeshDocumentSubscription(client=None, database=node.database, document=NOTE, key="note:bad") # type: ignore[arg-type]
|
||||||
|
|
||||||
|
def deliver(record: dict[str, Any]) -> Any:
|
||||||
|
message = {
|
||||||
|
"schemaVersion": "cultnet.database_change_raw.v0", "messageId": "change", "changeKind": "added",
|
||||||
|
"schemaId": SCHEMA_ID, "recordKey": record["recordKey"], "document": record,
|
||||||
|
}
|
||||||
|
subscription._subscription = _StubSubscription(CultNetDatabaseChange.from_wire(message)) # type: ignore[assignment]
|
||||||
|
return subscription.read_next_change()
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
deliver(record_of("note:bad", "bad", stamp))
|
||||||
|
self.assertIsNone(node.cache.get(NOTE, "note:bad"))
|
||||||
|
self.assertEqual(deliver(record_of("note:bad", "stamped")).value["text"], "stamped")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
1
packages/cultmesh-rs/Cargo.lock
generated
1
packages/cultmesh-rs/Cargo.lock
generated
|
|
@ -213,6 +213,7 @@ dependencies = [
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_stacker",
|
"serde_stacker",
|
||||||
|
"sha2",
|
||||||
"stacker",
|
"stacker",
|
||||||
"uuid",
|
"uuid",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
|
|
|
||||||
|
|
@ -1301,6 +1301,30 @@ mod tests {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The node's store is the second owner of the stamp rule: a snapshot document whose `storedAt` is no stamp refuses
|
||||||
|
/// the whole pull, nothing is minted or stored, and the node stays writable.
|
||||||
|
#[test]
|
||||||
|
fn a_snapshot_document_whose_stored_at_is_no_stamp_refuses_the_pull_and_stores_nothing() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
let path = temp.path().join("m.cc");
|
||||||
|
let mut node = CultMesh::create_node(&path, TestDocuments, Default::default())?;
|
||||||
|
let plain = |body: &str| rmp_serde::to_vec(&(body,));
|
||||||
|
for bad in ["", "yesterday", "2026-01-01T00:00:00", "2026-01-01t00:00:00z", "2026-01-01T00:00:60Z"] {
|
||||||
|
let mut stampless = raw_note("bad", plain("bad")?);
|
||||||
|
stampless.stored_at = bad.to_string();
|
||||||
|
let error = node
|
||||||
|
.apply_raw_snapshot_documents(vec![raw_note("good", plain("good")?), stampless])
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(format!("{error:#}").contains("stored_at"), "{bad:?}: {error:#}");
|
||||||
|
assert!(node.get::<Note>("good")?.is_none() && node.get::<Note>("bad")?.is_none(), "{bad:?}");
|
||||||
|
}
|
||||||
|
assert_eq!(node.apply_raw_snapshot_documents(vec![raw_note("after", plain("after")?)])?, 1);
|
||||||
|
let reopened = CultMesh::create_node(&path, TestDocuments, Default::default())?;
|
||||||
|
assert_eq!(reopened.get_required::<Note>("after")?.body, "after");
|
||||||
|
assert!(reopened.get::<Note>("good")?.is_none());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_snapshot_applied_to_a_stale_node_keeps_the_records_another_writer_added() -> Result<()> {
|
fn a_snapshot_applied_to_a_stale_node_keeps_the_records_another_writer_added() -> Result<()> {
|
||||||
let temp = tempfile::tempdir()?;
|
let temp = tempfile::tempdir()?;
|
||||||
|
|
|
||||||
|
|
@ -69,7 +69,8 @@ pub struct CultMeshRudpSnapshotQuery {
|
||||||
pub trait CultMeshRudpRawDocumentSink {
|
pub trait CultMeshRudpRawDocumentSink {
|
||||||
/// Whether this put may be written, decided without writing it. A receipt-capable session asks it only
|
/// Whether this put may be written, decided without writing it. A receipt-capable session asks it only
|
||||||
/// the first time it sees a message id; a resend is answered from what the session remembers
|
/// the first time it sees a message id; a resend is answered from what the session remembers
|
||||||
/// (write-receipt.md, rule 5). The default admits everything and leaves the decision to
|
/// (write-receipt.md, rule 5). A legacy session asks it only for a put whose `storedAt` is no stamp, so the
|
||||||
|
/// authority's refusal comes before the stamp's. The default admits everything and leaves the decision to
|
||||||
/// `accept_raw_document`.
|
/// `accept_raw_document`.
|
||||||
fn authorize_raw_document(&mut self, _receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> {
|
fn authorize_raw_document(&mut self, _receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -293,6 +294,9 @@ pub enum CultMeshRudpRejectionReason {
|
||||||
DocumentMissing,
|
DocumentMissing,
|
||||||
/// A receipt-capable session's put or delete has a null or missing `schemaId`.
|
/// A receipt-capable session's put or delete has a null or missing `schemaId`.
|
||||||
SchemaIdMissing,
|
SchemaIdMissing,
|
||||||
|
/// A put's `storedAt` is not a stamp (`cultcache_rs::is_stored_at_stamp`). It is judged after the sink's
|
||||||
|
/// authorization and before the document is measured or the sink commits it: nothing is minted or stored.
|
||||||
|
StoredAtNotStamp,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for CultMeshRudpRejectionReason {
|
impl std::fmt::Display for CultMeshRudpRejectionReason {
|
||||||
|
|
@ -307,6 +311,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason {
|
||||||
Self::ChannelNotReliable => f.write_str("a receipt session's write arrived on an unreliable channel"),
|
Self::ChannelNotReliable => f.write_str("a receipt session's write arrived on an unreliable channel"),
|
||||||
Self::DocumentMissing => f.write_str("the write has no document"),
|
Self::DocumentMissing => f.write_str("the write has no document"),
|
||||||
Self::SchemaIdMissing => f.write_str("the write has no schemaId"),
|
Self::SchemaIdMissing => f.write_str("the write has no schemaId"),
|
||||||
|
Self::StoredAtNotStamp => f.write_str("the document's storedAt is not a stamp"),
|
||||||
Self::DocumentUnservable {
|
Self::DocumentUnservable {
|
||||||
response_bytes,
|
response_bytes,
|
||||||
max_snapshot_response_bytes,
|
max_snapshot_response_bytes,
|
||||||
|
|
@ -347,7 +352,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason {
|
||||||
/// The texts a refused peer is sent, one per rejection variant. They are fixed:
|
/// The texts a refused peer is sent, one per rejection variant. They are fixed:
|
||||||
/// a sink's or source's error text can quote what a peer sent, so it stays with
|
/// a sink's or source's error text can quote what a peer sent, so it stays with
|
||||||
/// the caller that receives the rejection and never reaches the wire.
|
/// the caller that receives the rejection and never reaches the wire.
|
||||||
pub(crate) const REFUSAL_TEXTS: [&str; 17] = [
|
pub(crate) const REFUSAL_TEXTS: [&str; 18] = [
|
||||||
"the catalog refused the document",
|
"the catalog refused the document",
|
||||||
"the document can never be served",
|
"the document can never be served",
|
||||||
"the snapshot source failed",
|
"the snapshot source failed",
|
||||||
|
|
@ -365,6 +370,7 @@ pub(crate) const REFUSAL_TEXTS: [&str; 17] = [
|
||||||
"receipt sessions require the reliable channel",
|
"receipt sessions require the reliable channel",
|
||||||
"document is required",
|
"document is required",
|
||||||
"schemaId is required",
|
"schemaId is required",
|
||||||
|
"document must carry a storedAt stamp",
|
||||||
];
|
];
|
||||||
|
|
||||||
impl CultMeshRudpRejectionReason {
|
impl CultMeshRudpRejectionReason {
|
||||||
|
|
@ -393,7 +399,8 @@ impl CultMeshRudpRejectionReason {
|
||||||
13 => Self::MessageIdTooLong,
|
13 => Self::MessageIdTooLong,
|
||||||
14 => Self::ChannelNotReliable,
|
14 => Self::ChannelNotReliable,
|
||||||
15 => Self::DocumentMissing,
|
15 => Self::DocumentMissing,
|
||||||
_ => Self::SchemaIdMissing,
|
16 => Self::SchemaIdMissing,
|
||||||
|
_ => Self::StoredAtNotStamp,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -416,6 +423,7 @@ impl CultMeshRudpRejectionReason {
|
||||||
Self::ChannelNotReliable => 14,
|
Self::ChannelNotReliable => 14,
|
||||||
Self::DocumentMissing => 15,
|
Self::DocumentMissing => 15,
|
||||||
Self::SchemaIdMissing => 16,
|
Self::SchemaIdMissing => 16,
|
||||||
|
Self::StoredAtNotStamp => 17,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -498,6 +506,7 @@ impl CultMeshRudpApplicationRejection {
|
||||||
(CultMeshRudpRejectionReason::MessageIdTooLong, _) => message_invalid_message("messageId", text),
|
(CultMeshRudpRejectionReason::MessageIdTooLong, _) => message_invalid_message("messageId", text),
|
||||||
(CultMeshRudpRejectionReason::DocumentMissing, _) => message_invalid_message("document", text),
|
(CultMeshRudpRejectionReason::DocumentMissing, _) => message_invalid_message("document", text),
|
||||||
(CultMeshRudpRejectionReason::SchemaIdMissing, _) => message_invalid_message("schemaId", text),
|
(CultMeshRudpRejectionReason::SchemaIdMissing, _) => message_invalid_message("schemaId", text),
|
||||||
|
(CultMeshRudpRejectionReason::StoredAtNotStamp, _) => message_invalid_message("document", text),
|
||||||
(CultMeshRudpRejectionReason::ChannelNotReliable, _) => message_invalid_message("channel", text),
|
(CultMeshRudpRejectionReason::ChannelNotReliable, _) => message_invalid_message("channel", text),
|
||||||
(CultMeshRudpRejectionReason::WriteLimitReached, _) => CultNetMessage::Error {
|
(CultMeshRudpRejectionReason::WriteLimitReached, _) => CultNetMessage::Error {
|
||||||
error: format!("limit_reached: {text}"),
|
error: format!("limit_reached: {text}"),
|
||||||
|
|
@ -1601,11 +1610,19 @@ where
|
||||||
receipt: CultMeshRudpRawDocumentReceipt,
|
receipt: CultMeshRudpRawDocumentReceipt,
|
||||||
authorize: bool,
|
authorize: bool,
|
||||||
) -> std::result::Result<(), CultMeshRudpRejectionReason> {
|
) -> std::result::Result<(), CultMeshRudpRejectionReason> {
|
||||||
if authorize {
|
let stamped = cultcache_rs::is_stored_at_stamp(&receipt.document.stored_at);
|
||||||
|
// The authority speaks first. A legacy session's sink decides at `accept_raw_document`, so it is asked here
|
||||||
|
// only when the stamp is bad: an ungranted peer is refused `write_not_authorized`, not told its stamp is wrong.
|
||||||
|
if authorize || !stamped {
|
||||||
if let Err(error) = self.sink.authorize_raw_document(&receipt) {
|
if let Err(error) = self.sink.authorize_raw_document(&receipt) {
|
||||||
return Err(CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}")));
|
return Err(CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A peer's put is stored under the stamp it carries, so one that is no stamp is refused before anything is
|
||||||
|
// measured, minted or handed to the sink.
|
||||||
|
if !stamped {
|
||||||
|
return Err(CultMeshRudpRejectionReason::StoredAtNotStamp);
|
||||||
|
}
|
||||||
let document = &receipt.document;
|
let document = &receipt.document;
|
||||||
// Admit only what some snapshot request can return: the smallest
|
// Admit only what some snapshot request can return: the smallest
|
||||||
// response that could carry this document is the record the snapshot
|
// response that could carry this document is the record the snapshot
|
||||||
|
|
|
||||||
|
|
@ -3546,6 +3546,10 @@ fn blank_means_exactly_the_unicode_white_space_code_points_in_every_field() -> R
|
||||||
for (index, (hex, blank)) in space.iter().map(|h| (h, true)).chain(not_space.iter().map(|h| (h, false))).enumerate() {
|
for (index, (hex, blank)) in space.iter().map(|h| (h, true)).chain(not_space.iter().map(|h| (h, false))).enumerate() {
|
||||||
let hex = hex.as_str().unwrap();
|
let hex = hex.as_str().unwrap();
|
||||||
for (field_index, field) in fields.iter().enumerate() {
|
for (field_index, field) in fields.iter().enumerate() {
|
||||||
|
// A storedAt must be a stamp: a non-blank one that is no stamp is not a name (write-receipt.md, Required fields).
|
||||||
|
if !blank && *field == "storedAt" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
// A session that is sent refusals is allowed to end, so each code point and field gets its own.
|
// A session that is sent refusals is allowed to end, so each code point and field gets its own.
|
||||||
let sink = Sink::default();
|
let sink = Sink::default();
|
||||||
let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
|
let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
|
||||||
|
|
@ -3626,3 +3630,177 @@ fn an_unreadable_frame_on_a_receipt_connection_is_sent_exactly_the_fixed_text()
|
||||||
);
|
);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- A peer's `storedAt` must be a stamp (variants:ruling:raw-put-without-stamp-refused) ----
|
||||||
|
//
|
||||||
|
// `tests/vectors/stored-at-stamp/stamps.msgpack`: `[text, parses]` rows. Every row that is no stamp is refused and
|
||||||
|
// nothing reaches the sink. A blank text names nothing, so it is refused as a frame that does not read (field
|
||||||
|
// "message", write-receipt.md, Required fields); any other text that is no stamp is refused field "document".
|
||||||
|
|
||||||
|
fn stamp_rows() -> Vec<(String, bool)> {
|
||||||
|
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/vectors/stored-at-stamp/stamps.msgpack");
|
||||||
|
rmp_serde::from_slice(&std::fs::read(path).unwrap()).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_blank(text: &str) -> bool {
|
||||||
|
text.chars().all(char::is_whitespace)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A put of `key` whose `storedAt` is `stamp` as it would arrive: a blank one is swapped into the bytes, since the
|
||||||
|
/// encoder refuses it.
|
||||||
|
fn put_stamped_bytes(id: &str, key: &str, payload: Vec<u8>, stamp: &str) -> Result<Vec<u8>> {
|
||||||
|
let mut record = document(key, payload);
|
||||||
|
record.stored_at = if is_blank(stamp) { "zz-stamp-zz".into() } else { stamp.into() };
|
||||||
|
let bytes = encode_cultnet_message_to_vec(
|
||||||
|
&CultNetMessage::DocumentPutRaw { message_id: id.into(), document: record },
|
||||||
|
CultNetWireContract::CultNetSchemaV0,
|
||||||
|
)?;
|
||||||
|
Ok(if is_blank(stamp) { swap_fixstr(&bytes, "zz-stamp-zz", stamp) } else { bytes })
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_receipt_session_put_that_is_no_stamp_is_refused_for_every_vector_row_and_the_store_stays_writable() -> Result<()> {
|
||||||
|
let rows = stamp_rows();
|
||||||
|
assert!(rows.len() > 30);
|
||||||
|
let sink = Sink::default();
|
||||||
|
let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
|
||||||
|
let mut peer = client(server.local_addr()?, 700)?;
|
||||||
|
connect(&mut server, &mut [&mut peer])?;
|
||||||
|
send_reliable(&mut peer, &receipt_hello())?;
|
||||||
|
assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![server_hello()]);
|
||||||
|
let mut expected = Vec::new();
|
||||||
|
let mut answers = Vec::new();
|
||||||
|
for (index, (text, parses)) in rows.iter().enumerate() {
|
||||||
|
let id = format!("r{index}");
|
||||||
|
peer.send_reliable("schema", put_stamped_bytes(&id, &format!("note:{index}"), vec![1, 2, 3], text)?)?;
|
||||||
|
expected.push((id, text.clone(), *parses));
|
||||||
|
// One write in flight at a time: the answer arrives before the next row is sent.
|
||||||
|
answers.extend(frames_after(&mut server, &mut peer, 1)?);
|
||||||
|
}
|
||||||
|
peer.send_reliable("schema", valid_put("after"))?;
|
||||||
|
answers.extend(frames_after(&mut server, &mut peer, 1)?);
|
||||||
|
for (answer, (id, text, parses)) in answers.iter().zip(&expected) {
|
||||||
|
if *parses {
|
||||||
|
assert_eq!(*answer, admitted(id, &id.replace('r', "note:")), "{text:?}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::MessageInvalid), Some(id.clone())), "{text:?}");
|
||||||
|
let field = if is_blank(text) { "message" } else { "document" };
|
||||||
|
assert_eq!(error_text_and_field(answer).1, Some(field.into()), "{text:?}");
|
||||||
|
}
|
||||||
|
// Every stamp row was stored exactly as sent; no refused row was stored; the put after them all was.
|
||||||
|
let state = sink.0.lock().unwrap();
|
||||||
|
let stored: Vec<&str> = state.receipts.iter().map(|receipt| receipt.document.stored_at.as_str()).collect();
|
||||||
|
let stamps: Vec<&str> = rows.iter().filter(|(_, parses)| *parses).map(|(text, _)| text.as_str()).collect();
|
||||||
|
assert_eq!(&stored[..stamps.len()], &stamps[..]);
|
||||||
|
assert_eq!(stored.len(), stamps.len() + 1, "only the stamped puts and the final put were stored");
|
||||||
|
assert_eq!(answers.last(), Some(&admitted("put-after", "after")));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_put_with_no_storedat_key_is_refused_and_a_resend_replays_the_refusal() -> Result<()> {
|
||||||
|
let sink = Sink::default();
|
||||||
|
let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
|
||||||
|
let mut peer = client(server.local_addr()?, 701)?;
|
||||||
|
connect(&mut server, &mut [&mut peer])?;
|
||||||
|
send_reliable(&mut peer, &receipt_hello())?;
|
||||||
|
let missing = swap_fixstr(&valid_put("a"), "storedAt", "storedAx");
|
||||||
|
peer.send_reliable("schema", missing)?;
|
||||||
|
// The same garbage stamp twice under one id: the refusal is remembered (by index) and replays as it was.
|
||||||
|
peer.send_reliable("schema", put_stamped_bytes("g1", "note:g", vec![1], "yesterday")?)?;
|
||||||
|
peer.send_reliable("schema", put_stamped_bytes("g1", "note:g", vec![1], "yesterday")?)?;
|
||||||
|
let answers = answers_after(&mut server, &mut peer, 3)?;
|
||||||
|
assert_eq!(error_text_and_field(&answers[0]).1, Some("message".into()));
|
||||||
|
assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("g1".into())));
|
||||||
|
assert_eq!(error_text_and_field(&answers[1]).1, Some("document".into()));
|
||||||
|
assert_eq!(answers[2], answers[1]);
|
||||||
|
assert!(sink.0.lock().unwrap().receipts.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_legacy_session_put_that_is_no_stamp_is_refused_for_every_vector_row_and_nothing_is_stored() -> Result<()> {
|
||||||
|
let sink = Sink::default();
|
||||||
|
let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
|
||||||
|
let mut connection_id = 800;
|
||||||
|
for (text, parses) in stamp_rows() {
|
||||||
|
if parses {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
connection_id += 1;
|
||||||
|
let mut peer = client(server.local_addr()?, connection_id)?;
|
||||||
|
connect(&mut server, &mut [&mut peer])?;
|
||||||
|
let receipt = peer.send_reliable("schema", put_stamped_bytes("bad", "note:bad", vec![1, 2, 3], &text)?)?;
|
||||||
|
let reason = poll_until_rejected_or_stored(&mut server, &mut peer, &sink, 1)?;
|
||||||
|
if is_blank(&text) {
|
||||||
|
assert!(matches!(reason, Some(CultMeshRudpRejectionReason::MessageUnreadable(_))), "{text:?}: {reason:?}");
|
||||||
|
} else {
|
||||||
|
assert_eq!(reason, Some(CultMeshRudpRejectionReason::StoredAtNotStamp), "{text:?}");
|
||||||
|
let (error, code, details) = typed_refusal_seen_by(&mut peer, &receipt, Some(&mut server))?;
|
||||||
|
assert_eq!(code, Some(CultNetErrorCode::MessageInvalid), "{text:?}: {error}");
|
||||||
|
assert_eq!(details.and_then(|d| d.field), Some("document".into()), "{text:?}");
|
||||||
|
}
|
||||||
|
assert!(sink.0.lock().unwrap().receipts.is_empty(), "{text:?} reached the sink");
|
||||||
|
}
|
||||||
|
// The server and its sink are still writable.
|
||||||
|
connection_id += 1;
|
||||||
|
let mut peer = client(server.local_addr()?, connection_id)?;
|
||||||
|
connect(&mut server, &mut [&mut peer])?;
|
||||||
|
stored_after_poll(&mut server, &mut peer, &sink, 1)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The size gate measures the document as a snapshot would serve it: a put over the bound that is also no stamp is
|
||||||
|
/// refused for its stamp first, and nothing is measured for one that is.
|
||||||
|
#[test]
|
||||||
|
fn the_stamp_is_judged_before_the_size_limit() -> Result<()> {
|
||||||
|
let over = document("big", vec![7; 101]);
|
||||||
|
let options = CultMeshRudpDocumentServerOptions {
|
||||||
|
max_snapshot_response_bytes: served_alone_bytes(&over)? - 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let sink = Sink::default();
|
||||||
|
let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?;
|
||||||
|
let mut peer = client(server.local_addr()?, 900)?;
|
||||||
|
connect(&mut server, &mut [&mut peer])?;
|
||||||
|
peer.send_reliable("schema", put_stamped_bytes("o1", "big", vec![7; 101], "yesterday")?)?;
|
||||||
|
assert_eq!(
|
||||||
|
poll_until_rejected_or_stored(&mut server, &mut peer, &sink, 1)?,
|
||||||
|
Some(CultMeshRudpRejectionReason::StoredAtNotStamp)
|
||||||
|
);
|
||||||
|
let mut stamped = client(server.local_addr()?, 901)?;
|
||||||
|
connect(&mut server, &mut [&mut stamped])?;
|
||||||
|
stamped.send_reliable("schema", put_stamped_bytes("o2", "big", vec![7; 101], "2026-06-25T00:00:00Z")?)?;
|
||||||
|
assert!(matches!(
|
||||||
|
poll_until_rejected_or_stored(&mut server, &mut stamped, &sink, 1)?,
|
||||||
|
Some(CultMeshRudpRejectionReason::DocumentUnservable { .. })
|
||||||
|
));
|
||||||
|
assert!(sink.0.lock().unwrap().receipts.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An ungranted peer is refused `write_not_authorized` before its stamp is judged, on a receipt session and on a
|
||||||
|
/// legacy one, and the authority is asked about the write.
|
||||||
|
#[test]
|
||||||
|
fn an_ungranted_peer_is_refused_for_authority_before_its_stamp_is_judged() -> Result<()> {
|
||||||
|
let sink = Sink::denying(&["secret"]);
|
||||||
|
let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?;
|
||||||
|
let mut receipt_peer = client(server.local_addr()?, 950)?;
|
||||||
|
connect(&mut server, &mut [&mut receipt_peer])?;
|
||||||
|
send_reliable(&mut receipt_peer, &receipt_hello())?;
|
||||||
|
receipt_peer.send_reliable("schema", put_stamped_bytes("a1", "secret", vec![1], "yesterday")?)?;
|
||||||
|
let answers = answers_after(&mut server, &mut receipt_peer, 1)?;
|
||||||
|
assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::WriteNotAuthorized), Some("a1".into())));
|
||||||
|
assert_eq!(sink.0.lock().unwrap().authorized, vec!["a1".to_string()]);
|
||||||
|
|
||||||
|
let mut legacy_peer = client(server.local_addr()?, 951)?;
|
||||||
|
connect(&mut server, &mut [&mut legacy_peer])?;
|
||||||
|
legacy_peer.send_reliable("schema", put_stamped_bytes("a2", "secret", vec![1], "yesterday")?)?;
|
||||||
|
assert!(matches!(
|
||||||
|
poll_until_rejected_or_stored(&mut server, &mut legacy_peer, &sink, 1)?,
|
||||||
|
Some(CultMeshRudpRejectionReason::SinkRefused(_))
|
||||||
|
));
|
||||||
|
assert!(sink.0.lock().unwrap().receipts.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
|
- `createRudpDocumentServer` refuses a peer put whose `storedAt` is missing, empty or not a stamp (`isStoredAtStamp`, the shape pinned in `tests/vectors/stored-at-stamp/`) with `message_invalid` naming `document`, after the write authority has been asked and before the put is sized; `onDocumentPutRaw` is not called, so nothing is stored and no stamp is minted (variants:ruling:raw-put-without-stamp-refused). `CultMeshRudpDocumentPut.storedAt` is the stamp the peer sent, never a server clock; a document synced from a peer with a stampless record is refused by the registry.
|
||||||
- `publishRudpDocumentOnce` no longer treats a window with no refusal as published: a put nothing settled in its result window (500 ms, or 5000 ms against a receipt-capable server) rejects `CultMeshRudpWriteUnconfirmedError`, and a legacy server's echo of the put's id is not admission. It now resolves `"admitted"` (only on a `write_admitted`) or `"sent"`. **Migration:** a caller whose server cannot answer a write yet (for example Odin, until it is receipt-capable) passes `sendOnly: true` per call and treats `"sent"` as "sent", never as admitted. Receipts stay opt-in (`writeReceipts: true`); neither mode is the default.
|
- `publishRudpDocumentOnce` no longer treats a window with no refusal as published: a put nothing settled in its result window (500 ms, or 5000 ms against a receipt-capable server) rejects `CultMeshRudpWriteUnconfirmedError`, and a legacy server's echo of the put's id is not admission. It now resolves `"admitted"` (only on a `write_admitted`) or `"sent"`. **Migration:** a caller whose server cannot answer a write yet (for example Odin, until it is receipt-capable) passes `sendOnly: true` per call and treats `"sent"` as "sent", never as admitted. Receipts stay opt-in (`writeReceipts: true`); neither mode is the default.
|
||||||
- The package is renamed `cultmesh-ts` to `@gamecult/cultmesh-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-ts` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`.
|
- The package is renamed `cultmesh-ts` to `@gamecult/cultmesh-ts` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-ts` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`.
|
||||||
- `asSchemaAlias` is removed from `CultMeshDocumentHandle`, `CultMeshCollectionHandle` and their bound forms. Read the handle by the schema you mean.
|
- `asSchemaAlias` is removed from `CultMeshDocumentHandle`, `CultMeshCollectionHandle` and their bound forms. Read the handle by the schema you mean.
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,7 @@
|
||||||
"node": ">=20"
|
"node": ">=20"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@gamecult/cultcache-ts": "^0.15.0",
|
"@gamecult/cultcache-ts": "^0.16.0",
|
||||||
"@gamecult/cultnet-ts": "^0.4.0",
|
"@gamecult/cultnet-ts": "^0.4.0",
|
||||||
"koffi": "3.3.0"
|
"koffi": "3.3.0"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import {
|
||||||
CultCache,
|
CultCache,
|
||||||
SchemaConflictError,
|
SchemaConflictError,
|
||||||
SingleFileMessagePackBackingStore,
|
SingleFileMessagePackBackingStore,
|
||||||
|
isStoredAtStamp,
|
||||||
schemaIdentityOf,
|
schemaIdentityOf,
|
||||||
type AnyCultCacheDocumentDefinition,
|
type AnyCultCacheDocumentDefinition,
|
||||||
type CacheBackingStore,
|
type CacheBackingStore,
|
||||||
|
|
@ -5813,6 +5814,12 @@ export class CultMesh {
|
||||||
));
|
));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// A peer's put is stored under the stamp it carries: one that is no stamp is refused, after the authority
|
||||||
|
// has spoken and before the put is sized, so the size-limited path never measures it. Nothing is minted.
|
||||||
|
if (!isStoredAtStamp(message.document.storedAt)) {
|
||||||
|
sendSchemaMessage(record, cultNetMessageInvalidError("document", "Raw put documents must carry a storedAt stamp."));
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (options.onDocumentPutRaw) {
|
if (options.onDocumentPutRaw) {
|
||||||
const unservable = unservablePut(message);
|
const unservable = unservablePut(message);
|
||||||
|
|
@ -6635,7 +6642,7 @@ function normalizeRudpDocumentPut(
|
||||||
return {
|
return {
|
||||||
schemaId: document.schemaId,
|
schemaId: document.schemaId,
|
||||||
recordKey: document.recordKey,
|
recordKey: document.recordKey,
|
||||||
storedAt: document.storedAt ?? new Date().toISOString(),
|
storedAt: document.storedAt,
|
||||||
sourceRuntimeId: document.sourceRuntimeId ?? null,
|
sourceRuntimeId: document.sourceRuntimeId ?? null,
|
||||||
sourceAgentId: document.sourceAgentId ?? null,
|
sourceAgentId: document.sourceAgentId ?? null,
|
||||||
sourceRole: document.sourceRole ?? null,
|
sourceRole: document.sourceRole ?? null,
|
||||||
|
|
|
||||||
|
|
@ -3423,11 +3423,12 @@ const servedBoundRegistry = () =>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A put's own storedAt, kept by the served record and replaced in the receipt by the server's
|
* A put's own storedAt, kept by the served record and replaced in the receipt by the server's
|
||||||
* 24-character clock. `SERVED_STORED_AT` is long, so an echoed receipt is smaller than the served
|
* 24-character clock. `SERVED_STORED_AT` is as long as a stamp can be (a stamp is judged before a put is
|
||||||
* reply; `RECEIPT_STORED_AT` is short, so an echoed receipt is the larger reply, which shows that
|
* sized, so padding one is refused), so an echoed receipt is smaller than the served reply;
|
||||||
|
* `RECEIPT_STORED_AT` is short, so an echoed receipt is the larger reply, which shows that
|
||||||
* admission sizes only the served reply.
|
* admission sizes only the served reply.
|
||||||
*/
|
*/
|
||||||
const SERVED_STORED_AT = `2026-09-30T00:00:00.000Z${"~".repeat(200)}`;
|
const SERVED_STORED_AT = "2026-09-30T00:00:00.123456789+00:00"; // the longest a stamp can be, 35 characters
|
||||||
const RECEIPT_STORED_AT = "2026-09-30T00:00:00.000Z";
|
const RECEIPT_STORED_AT = "2026-09-30T00:00:00.000Z";
|
||||||
const SERVED_BOUND_SERVER = "cultmesh-ts-served-bound-server";
|
const SERVED_BOUND_SERVER = "cultmesh-ts-served-bound-server";
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -122,7 +122,7 @@ const rust: Target = {
|
||||||
name: "rust",
|
name: "rust",
|
||||||
payload: (text) => encode({ text }),
|
payload: (text) => encode({ text }),
|
||||||
async start() {
|
async start() {
|
||||||
if (!prebuilt) await execFileAsync(cargo, ["build", "--example", "receipt_interop_server", "--message-format", "short"], { cwd: rustManifestDir });
|
if (!prebuilt) await execFileAsync(cargo, ["build", "--locked", "--example", "receipt_interop_server", "--message-format", "short"], { cwd: rustManifestDir });
|
||||||
const metadata = spawnSync(cargo, ["metadata", "--format-version", "1", "--no-deps"], { cwd: rustManifestDir, encoding: "utf8" });
|
const metadata = spawnSync(cargo, ["metadata", "--format-version", "1", "--no-deps"], { cwd: rustManifestDir, encoding: "utf8" });
|
||||||
if (metadata.status !== 0) throw new Error(`cargo metadata failed: ${metadata.stderr}`);
|
if (metadata.status !== 0) throw new Error(`cargo metadata failed: ${metadata.stderr}`);
|
||||||
const exe = join(JSON.parse(metadata.stdout).target_directory, "debug", "examples", `receipt_interop_server${process.platform === "win32" ? ".exe" : ""}`);
|
const exe = join(JSON.parse(metadata.stdout).target_directory, "debug", "examples", `receipt_interop_server${process.platform === "win32" ? ".exe" : ""}`);
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import { join } from "node:path";
|
||||||
import test from "node:test";
|
import test from "node:test";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { documentPayloadFromBytes, defineDocumentType } from "@gamecult/cultcache-ts";
|
import { documentPayloadFromBytes, defineDocumentType } from "@gamecult/cultcache-ts";
|
||||||
|
import { decode } from "@msgpack/msgpack";
|
||||||
import { decodeRudpPacket } from "@gamecult/cultnet-ts";
|
import { decodeRudpPacket } from "@gamecult/cultnet-ts";
|
||||||
import { encodeWireValue } from "@gamecult/cultnet-ts/wire";
|
import { encodeWireValue } from "@gamecult/cultnet-ts/wire";
|
||||||
import {
|
import {
|
||||||
|
|
@ -310,6 +311,64 @@ test("the refusal on the wire is the pinned error-vectors bytes", async () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// A peer's put is stored under the stamp it carries: every row of the shared vector goes through the real handler. A row that
|
||||||
|
// is no stamp is refused message_invalid naming `document`, on the plain, the size-limited and (RUDP is this server's only
|
||||||
|
// transport) the capped path; the handler never runs, so nothing is stored or minted, and a stamped put after it still lands.
|
||||||
|
const STAMP_VECTOR = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "stored-at-stamp", "stamps.msgpack");
|
||||||
|
|
||||||
|
test("a put whose storedAt is no stamp is refused message_invalid (document), for every row of the stamp vector", async () => {
|
||||||
|
const rows = decode(readFileSync(STAMP_VECTOR)) as [string, boolean][];
|
||||||
|
const cases: [string, unknown, boolean][] = [...rows.map(([text, parses]): [string, unknown, boolean] => [JSON.stringify(text), text, parses]),
|
||||||
|
["missing", undefined, false], ["number", 1_700_000_000_000, false]];
|
||||||
|
for (const extra of [{}, { maxPayloadBytes: 4096 }]) {
|
||||||
|
await using(await rig(new CultMeshGrantAllWrites(), extra), async (r) => {
|
||||||
|
let accepted = 0;
|
||||||
|
for (const [label, storedAt, parses] of cases) {
|
||||||
|
const message = put(KEY_A) as unknown as { document: Record<string, unknown> };
|
||||||
|
if (storedAt === undefined) delete message.document.storedAt; else message.document.storedAt = storedAt;
|
||||||
|
const seen = r.replies.length;
|
||||||
|
const handled = r.puts.length;
|
||||||
|
r.peer.send(message as unknown as CultNetMessage);
|
||||||
|
await sleep(40);
|
||||||
|
if (parses) {
|
||||||
|
accepted += 1;
|
||||||
|
assert.equal(r.puts.length, handled + 1, `${label}: a stamp reaches the handler`);
|
||||||
|
assert.equal(r.replies.length, seen, label);
|
||||||
|
} else {
|
||||||
|
assert.equal(r.puts.length, handled, `${label}: nothing handled, so nothing minted or stored`);
|
||||||
|
const reply = r.replies[seen] as { schemaVersion?: string; code?: string; details?: { field?: string } } | undefined;
|
||||||
|
assert.equal(reply?.schemaVersion, "cultnet.error.v0", label);
|
||||||
|
assert.equal(reply?.code, "message_invalid", label);
|
||||||
|
assert.equal(reply?.details?.field, "document", label);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.ok(accepted > 0);
|
||||||
|
assert.deepEqual(r.errors, []);
|
||||||
|
// The large put under a garbage stamp is refused for its stamp, before it is measured.
|
||||||
|
const large = put(KEY_A, {}, SCHEMA_ID, new Uint8Array(20_000)) as unknown as { document: Record<string, unknown> };
|
||||||
|
large.document.storedAt = "yesterday";
|
||||||
|
r.peer.send(large as unknown as CultNetMessage);
|
||||||
|
await sleep(60);
|
||||||
|
const last = r.replies[r.replies.length - 1] as { code?: string; details?: { field?: string } };
|
||||||
|
assert.equal(last.code, "message_invalid");
|
||||||
|
assert.equal(last.details?.field, "document");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// The stamp is judged after the authority: an ungranted peer hears write_not_authorized, aliased or not.
|
||||||
|
const refusing = new Recording(() => ({ granted: false, reason: "no" }));
|
||||||
|
await using(await rig(refusing), async (r) => {
|
||||||
|
const aliased = put(KEY_A, {}, ALIAS) as unknown as { document: Record<string, unknown> };
|
||||||
|
aliased.document.storedAt = "yesterday";
|
||||||
|
r.peer.send(aliased as unknown as CultNetMessage);
|
||||||
|
r.peer.send(put(KEY_B));
|
||||||
|
await r.settle();
|
||||||
|
assertRefused(r.replies[0], KEY_A);
|
||||||
|
assertRefused(r.replies[1], KEY_B);
|
||||||
|
assert.equal(refusing.requests[0]?.schemaId, SCHEMA_ID, "an aliased put asks under the canonical id");
|
||||||
|
assert.equal(r.puts.length, 0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test("wrong-typed shard fields on a put are judged with null shard fields, then refused as malformed once granted", async () => {
|
test("wrong-typed shard fields on a put are judged with null shard fields, then refused as malformed once granted", async () => {
|
||||||
const authority = new Recording(grant);
|
const authority = new Recording(grant);
|
||||||
await using(await rig(authority), async (r) => {
|
await using(await rig(authority), async (r) => {
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
## Unreleased (breaking; version 0.x, so the next minor)
|
## Unreleased (breaking; version 0.x, so the next minor)
|
||||||
|
|
||||||
|
- `raw_record_to_envelope`, where a replicated record becomes an envelope, refuses a record whose `storedAt` is missing, empty or not a stamp (`cultcache_py.stores.parse_stored_at`, the shape pinned by `tests/vectors/stored-at-stamp`) with `ValueError`, nothing minted or stored (`variants:ruling:raw-put-without-stamp-refused`). `apply_raw_document_record`, `apply_raw_snapshot` and `apply_shard_log_response` come through it: a snapshot response with such a record is refused whole before any of its records is stored, as one naming an unregistered schema is; a shard log refuses at that entry, the entries before it already applied. It stored the record under `""`, and the next open of the store raised `CultCacheError` `Missing envelope`. `cultmesh-py`'s raw put builds its envelope through the same function, so there is one owner of the check there; its server gate still answers a peer's put `message_invalid`. A client's `read_next_change` lets the `ValueError` out.
|
||||||
- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). `resolve_document_for_raw_record` reads the schema hint from a raw record's first value whether or not bytes follow it, so a record routed by its stamp is refused by count when applied (`ValueError`, in a snapshot, a shard log and a direct apply) like one routed by a registered schema id; `apply_shard_log_response` used to skip it silently as an unknown schema. A tail on a payload whose first value names no registered schema is still `KeyError`, which a shard log skips.
|
- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). `resolve_document_for_raw_record` reads the schema hint from a raw record's first value whether or not bytes follow it, so a record routed by its stamp is refused by count when applied (`ValueError`, in a snapshot, a shard log and a direct apply) like one routed by a registered schema id; `apply_shard_log_response` used to skip it silently as an unknown schema. A tail on a payload whose first value names no registered schema is still `KeyError`, which a shard log skips.
|
||||||
- `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out.
|
- `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out.
|
||||||
- `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move.
|
- `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move.
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ from typing import Any, Iterable
|
||||||
|
|
||||||
from cultcache_py import CultCache, CultCacheEnvelope
|
from cultcache_py import CultCache, CultCacheEnvelope
|
||||||
from cultcache_py.documents import DocumentDefinition
|
from cultcache_py.documents import DocumentDefinition
|
||||||
|
from cultcache_py.stores import parse_stored_at
|
||||||
|
|
||||||
from .shard_log import CultNetShardLogResponse
|
from .shard_log import CultNetShardLogResponse
|
||||||
from .snapshot import CultNetRawDocumentRecord, CultNetRawSnapshotResponse
|
from .snapshot import CultNetRawDocumentRecord, CultNetRawSnapshotResponse
|
||||||
|
|
@ -50,17 +51,29 @@ def apply_raw_document_record(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def require_stored_at_stamp(record: dict[str, Any]) -> str:
|
||||||
|
"""The storedAt a replicated raw record carries; ValueError when it is missing or no stamp. The one place the rule is judged."""
|
||||||
|
stored_at = record.get("storedAt")
|
||||||
|
if parse_stored_at(stored_at) is None:
|
||||||
|
raise ValueError("A replicated raw record must carry a storedAt stamp.")
|
||||||
|
return stored_at
|
||||||
|
|
||||||
|
|
||||||
def raw_record_to_envelope(
|
def raw_record_to_envelope(
|
||||||
document: DocumentDefinition[Any],
|
document: DocumentDefinition[Any],
|
||||||
schema_id: str,
|
schema_id: str,
|
||||||
record: dict[str, Any],
|
record: dict[str, Any],
|
||||||
) -> CultCacheEnvelope:
|
) -> CultCacheEnvelope:
|
||||||
|
"""The envelope a replicated raw record is stored as; the one place a record becomes an envelope (a direct apply, a snapshot and a
|
||||||
|
shard log all come through here). A peer's record is stored under the stamp it carries, so one whose storedAt is missing, empty or
|
||||||
|
no stamp is refused with ValueError and nothing is minted or stored (variants:ruling:raw-put-without-stamp-refused)."""
|
||||||
|
stored_at = require_stored_at_stamp(record)
|
||||||
return CultCacheEnvelope(
|
return CultCacheEnvelope(
|
||||||
key=str(record["recordKey"]),
|
key=str(record["recordKey"]),
|
||||||
type=document.type,
|
type=document.type,
|
||||||
schema_id=schema_id,
|
schema_id=schema_id,
|
||||||
payload=bytes(record["payload"]),
|
payload=bytes(record["payload"]),
|
||||||
stored_at=str(record.get("storedAt") or ""),
|
stored_at=stored_at,
|
||||||
catalog_entry=document.catalog_entry(),
|
catalog_entry=document.catalog_entry(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -117,6 +130,12 @@ def apply_shard_log_response(
|
||||||
reason = response.get("reason") or "unspecified"
|
reason = response.get("reason") or "unspecified"
|
||||||
raise ValueError(f"Shard log response requires resync: {reason}")
|
raise ValueError(f"Shard log response requires resync: {reason}")
|
||||||
documents_by_schema_id = schema_document_map(documents)
|
documents_by_schema_id = schema_document_map(documents)
|
||||||
|
# The whole log is refused before anything is stored: every put that would be applied is judged for its stamp first.
|
||||||
|
for entry in response.get("entries", []):
|
||||||
|
if isinstance(entry, dict) and entry.get("changeKind") in ("added", "updated") and isinstance(entry.get("put"), dict):
|
||||||
|
record = entry["put"]["document"]
|
||||||
|
if can_resolve_document_for_raw_record(documents_by_schema_id, str(record["schemaId"]), record):
|
||||||
|
require_stored_at_stamp(record)
|
||||||
applied: list[CultNetAppliedRecord] = []
|
applied: list[CultNetAppliedRecord] = []
|
||||||
for entry in response.get("entries", []):
|
for entry in response.get("entries", []):
|
||||||
if not isinstance(entry, dict):
|
if not isinstance(entry, dict):
|
||||||
|
|
|
||||||
57
packages/cultnet-py/tests/test_raw_record_stamp.py
Normal file
57
packages/cultnet-py/tests/test_raw_record_stamp.py
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
"""raw_record_to_envelope is the one place a replicated raw record becomes an envelope; it refuses a record whose storedAt is no stamp.
|
||||||
|
|
||||||
|
The stamp shape is tests/vectors/stored-at-stamp/stamps.msgpack (variants:ruling:raw-put-without-stamp-refused); cultmesh-py tests the
|
||||||
|
whole replication paths, this tests the function itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
from cultcache_py import define_database_entry_type
|
||||||
|
from cultnet_py.replication import raw_record_to_envelope
|
||||||
|
|
||||||
|
VECTOR = Path(__file__).resolve().parents[3] / "tests/vectors/stored-at-stamp/stamps.msgpack"
|
||||||
|
ROWS: list[tuple[str, bool]] = [(text, parses) for text, parses in msgpack.unpackb(VECTOR.read_bytes(), raw=False)]
|
||||||
|
NOTE = define_database_entry_type(
|
||||||
|
"tests.envelope_note",
|
||||||
|
[("schema_version", 0), ("text", 1)],
|
||||||
|
schema_id="sha256:envelope-note",
|
||||||
|
schema_name="tests.envelope_note",
|
||||||
|
schema_version="tests.envelope_note.v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def record_with(stamp: Any) -> dict[str, Any]:
|
||||||
|
record = {
|
||||||
|
"schemaId": "sha256:envelope-note",
|
||||||
|
"recordKey": "note:1",
|
||||||
|
"payload": NOTE.encode_payload({"schema_version": "tests.envelope_note.v1", "text": "t"}),
|
||||||
|
}
|
||||||
|
if stamp is not None:
|
||||||
|
record["storedAt"] = stamp
|
||||||
|
return record
|
||||||
|
|
||||||
|
|
||||||
|
class RawRecordToEnvelopeStampTests(unittest.TestCase):
|
||||||
|
def test_every_vector_row_is_an_envelope_or_a_refusal_as_the_vector_says(self) -> None:
|
||||||
|
self.assertGreater(len(ROWS), 30)
|
||||||
|
for text, parses in ROWS:
|
||||||
|
with self.subTest(text=text):
|
||||||
|
if parses:
|
||||||
|
self.assertEqual(raw_record_to_envelope(NOTE, "sha256:envelope-note", record_with(text)).stored_at, text)
|
||||||
|
else:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
raw_record_to_envelope(NOTE, "sha256:envelope-note", record_with(text))
|
||||||
|
|
||||||
|
def test_a_missing_or_non_string_stamp_is_refused(self) -> None:
|
||||||
|
for stamp in (None, 12345, b"2026-11-07T00:00:00Z"):
|
||||||
|
with self.subTest(stamp=stamp), self.assertRaises(ValueError):
|
||||||
|
raw_record_to_envelope(NOTE, "sha256:envelope-note", record_with(stamp))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
1
packages/cultnet-rs/.gitignore
vendored
1
packages/cultnet-rs/.gitignore
vendored
|
|
@ -1,2 +1 @@
|
||||||
/target
|
/target
|
||||||
Cargo.lock
|
|
||||||
|
|
|
||||||
1228
packages/cultnet-rs/Cargo.lock
generated
Normal file
1228
packages/cultnet-rs/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -30,7 +30,7 @@ knock on the contract before touching the furniture.
|
||||||
## Receipts
|
## Receipts
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
cargo test
|
cargo test --locked
|
||||||
```
|
```
|
||||||
|
|
||||||
The initial tests prove:
|
The initial tests prove:
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@
|
||||||
//! traffic crosses the same schedule. The schedule is `policy.rs`.
|
//! traffic crosses the same schedule. The schedule is `policy.rs`.
|
||||||
//!
|
//!
|
||||||
//! ```text
|
//! ```text
|
||||||
//! cargo run --release --example cultnet_impair -- \
|
//! cargo run --locked --release --example cultnet_impair -- \
|
||||||
//! --listen 0.0.0.0:17890 --upstream 127.0.0.1:17990 \
|
//! --listen 0.0.0.0:17890 --upstream 127.0.0.1:17990 \
|
||||||
//! --profile tests/impairment/loss-1pct.toml --seed 424242 --metrics loss-1pct.csv
|
//! --profile tests/impairment/loss-1pct.toml --seed 424242 --metrics loss-1pct.csv
|
||||||
//! ```
|
//! ```
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,7 @@ fn write_record(path: &Path, key: &str, ty: &str, schema: &str, payload: Vec<u8>
|
||||||
stored_at: "2026-10-29T00:00:00Z".into(),
|
stored_at: "2026-10-29T00:00:00Z".into(),
|
||||||
schema_id: Some(schema.into()),
|
schema_id: Some(schema.into()),
|
||||||
})
|
})
|
||||||
|
.map(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------------ fake host
|
// ------------------------------------------------------------------ fake host
|
||||||
|
|
|
||||||
|
|
@ -347,7 +347,9 @@ pub fn export_service_identity_trust_anchor<P: ServiceIdentityProfile>(
|
||||||
};
|
};
|
||||||
let mut backing = SingleFileMessagePackBackingStore::new(path);
|
let mut backing = SingleFileMessagePackBackingStore::new(path);
|
||||||
match backing.pull_all()?.as_slice() {
|
match backing.pull_all()?.as_slice() {
|
||||||
[] => backing.push(&envelope)?,
|
[] => {
|
||||||
|
backing.push(&envelope)?;
|
||||||
|
}
|
||||||
[current] if current == &envelope => {}
|
[current] if current == &envelope => {}
|
||||||
_ => bail!("service identity trust anchor output already contains different state"),
|
_ => bail!("service identity trust anchor output already contains different state"),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
//! Ignored by default; run in release, where the numbers mean something:
|
//! Ignored by default; run in release, where the numbers mean something:
|
||||||
//!
|
//!
|
||||||
//! ```text
|
//! ```text
|
||||||
//! cargo test --release --test media_fec_bench -- --ignored --nocapture
|
//! cargo test --locked --release --test media_fec_bench -- --ignored --nocapture
|
||||||
//! ```
|
//! ```
|
||||||
//!
|
//!
|
||||||
//! Each line is nanoseconds per call through the public surface, so record
|
//! Each line is nanoseconds per call through the public surface, so record
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ All notable changes to `@gamecult/cultnet-ts` are recorded here, following
|
||||||
|
|
||||||
### Breaking
|
### Breaking
|
||||||
|
|
||||||
|
- `CultNetDocumentRegistry.applyRawDocumentPutMessage` and `createRawSnapshotResponseForPut` throw `A raw put must carry a storedAt stamp.` when the put's `storedAt` is missing, empty or not a stamp (`isStoredAtStamp` of cultcache-ts), and nothing is stored. The parser no longer refuses a `cultnet.document_put_raw.v0` for its `storedAt`: the server and the node judge it, so a refusal can name `document` (snapshot records and database changes are still read strictly as non-empty).
|
||||||
- `decodeManifest` refuses a manifest with bytes after its first MessagePack value (`CDN artifact manifest has N bytes after its first MessagePack value.`), as the C# reference does; before, the tail was ignored. `contracts/cultmesh/content-acceptance-vectors.json` now holds `trailing_nil`, `trailing_reserved_code` and `trailing_truncated_array` as refused.
|
- `decodeManifest` refuses a manifest with bytes after its first MessagePack value (`CDN artifact manifest has N bytes after its first MessagePack value.`), as the C# reference does; before, the tail was ignored. `contracts/cultmesh/content-acceptance-vectors.json` now holds `trailing_nil`, `trailing_reserved_code` and `trailing_truncated_array` as refused.
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,7 @@
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.3",
|
"@msgpack/msgpack": "^3.1.3",
|
||||||
"ajv": "^8.17.1",
|
"ajv": "^8.17.1",
|
||||||
"@gamecult/cultcache-ts": "^0.15.0"
|
"@gamecult/cultcache-ts": "^0.16.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^24.0.0",
|
"@types/node": "^24.0.0",
|
||||||
|
|
|
||||||
|
|
@ -1289,7 +1289,9 @@ function validateDocumentPutRawMessage(input: unknown): asserts input is CultNet
|
||||||
throw new Error("Validation failed for cultnet.document_put_raw.v0: /document: is required");
|
throw new Error("Validation failed for cultnet.document_put_raw.v0: /document: is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
validateRawDocumentRecord(candidate.document, "/document");
|
// A peer's put is judged for its stamp by the server and the node (isStoredAtStamp), which answer message_invalid naming
|
||||||
|
// `document` after the write authority has spoken; the parser must let such a put through to them.
|
||||||
|
validateRawDocumentRecord(candidate.document, "/document", { judgesStoredAt: false });
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateSnapshotResponseRawMessage(
|
function validateSnapshotResponseRawMessage(
|
||||||
|
|
@ -1450,6 +1452,7 @@ function validateBinaryMessageRoot(
|
||||||
function validateRawDocumentRecord(
|
function validateRawDocumentRecord(
|
||||||
input: unknown,
|
input: unknown,
|
||||||
instancePath: string,
|
instancePath: string,
|
||||||
|
{ judgesStoredAt = true }: { judgesStoredAt?: boolean } = {},
|
||||||
): asserts input is CultNetRawDocumentRecord {
|
): asserts input is CultNetRawDocumentRecord {
|
||||||
if (!input || typeof input !== "object") {
|
if (!input || typeof input !== "object") {
|
||||||
throw new Error(`Validation failed for raw CultNet document record: ${instancePath}: must be an object`);
|
throw new Error(`Validation failed for raw CultNet document record: ${instancePath}: must be an object`);
|
||||||
|
|
@ -1461,7 +1464,9 @@ function validateRawDocumentRecord(
|
||||||
requireOptionalNonEmptyString(candidate.schemaVersion, `${instancePath}/schemaVersion`);
|
requireOptionalNonEmptyString(candidate.schemaVersion, `${instancePath}/schemaVersion`);
|
||||||
requireOptionalNonEmptyString(candidate.schemaContentHash, `${instancePath}/schemaContentHash`);
|
requireOptionalNonEmptyString(candidate.schemaContentHash, `${instancePath}/schemaContentHash`);
|
||||||
requireNonEmptyString(candidate.recordKey, `${instancePath}/recordKey`);
|
requireNonEmptyString(candidate.recordKey, `${instancePath}/recordKey`);
|
||||||
requireNonEmptyString(candidate.storedAt, `${instancePath}/storedAt`);
|
if (judgesStoredAt) {
|
||||||
|
requireNonEmptyString(candidate.storedAt, `${instancePath}/storedAt`);
|
||||||
|
}
|
||||||
if (candidate.payloadEncoding !== "messagepack") {
|
if (candidate.payloadEncoding !== "messagepack") {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Validation failed for raw CultNet document record: ${instancePath}/payloadEncoding: must equal messagepack`,
|
`Validation failed for raw CultNet document record: ${instancePath}/payloadEncoding: must equal messagepack`,
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import type {
|
||||||
} from "@gamecult/cultcache-ts";
|
} from "@gamecult/cultcache-ts";
|
||||||
import {
|
import {
|
||||||
SchemaConflictError,
|
SchemaConflictError,
|
||||||
|
isStoredAtStamp,
|
||||||
decodeDocumentPayload,
|
decodeDocumentPayload,
|
||||||
documentPayloadBytes,
|
documentPayloadBytes,
|
||||||
documentPayloadFromBytes,
|
documentPayloadFromBytes,
|
||||||
|
|
@ -323,6 +324,10 @@ export class CultNetDocumentRegistry {
|
||||||
envelope: CultCacheEnvelope;
|
envelope: CultCacheEnvelope;
|
||||||
} {
|
} {
|
||||||
const binding = this.#requireSchemaBinding(message.document.schemaId);
|
const binding = this.#requireSchemaBinding(message.document.schemaId);
|
||||||
|
// A put is stored under the stamp it carries; one that is no stamp is refused, and nothing is minted for it.
|
||||||
|
if (!isStoredAtStamp(message.document.storedAt)) {
|
||||||
|
throw new Error("A raw put must carry a storedAt stamp.");
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
binding,
|
binding,
|
||||||
envelope: {
|
envelope: {
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ cultnet-rs only, and Idunn's `IdunnRuntimeActivationLaunch` is that same crate's
|
||||||
It needs Linux and a machine-id equal to the `machine-id` file here, because the
|
It needs Linux and a machine-id equal to the `machine-id` file here, because the
|
||||||
provider identity is bound to it. On Yggdrasil:
|
provider identity is bound to it. On Yggdrasil:
|
||||||
|
|
||||||
DOCKER_ARGS='--cap-add SYS_ADMIN --security-opt apparmor=unconfined' bash ygg-verify.sh <CultLib> HEAD rust 'cd packages/cultnet-rs && cp <fixed-machine-id> /tmp/mid && mount --bind /tmp/mid /etc/machine-id && cargo run -q --example idunn_runtime_fixture /tmp/out'
|
DOCKER_ARGS='--cap-add SYS_ADMIN --security-opt apparmor=unconfined' bash ygg-verify.sh <CultLib> HEAD rust 'cd packages/cultnet-rs && cp <fixed-machine-id> /tmp/mid && mount --bind /tmp/mid /etc/machine-id && cargo run --locked -q --example idunn_runtime_fixture /tmp/out'
|
||||||
|
|
||||||
then copy `/tmp/out` over this directory. `--cap-add SYS_ADMIN` exists only so the bind mount can present the committed `machine-id`, which keeps Yggdrasil's real machine-id out of the repo. Every `presence-*.bin` was accepted by
|
then copy `/tmp/out` over this directory. `--cap-add SYS_ADMIN` exists only so the bind mount can present the committed `machine-id`, which keeps Yggdrasil's real machine-id out of the repo. Every `presence-*.bin` was accepted by
|
||||||
`authenticate_runtime_presence_claim` and correlated by
|
`authenticate_runtime_presence_claim` and correlated by
|
||||||
|
|
@ -26,7 +26,7 @@ The generator takes its specification as input, so a consumer needs no patch to
|
||||||
specification is command-line flags; `@<file>` splices in a response file (one `--flag value`
|
specification is command-line flags; `@<file>` splices in a response file (one `--flag value`
|
||||||
per line, `#` comments). CultLib's own sets are `fixture.args` here:
|
per line, `#` comments). CultLib's own sets are `fixture.args` here:
|
||||||
|
|
||||||
cargo run -q --example idunn_runtime_fixture /tmp/out @<path>/fixture.args
|
cargo run --locked -q --example idunn_runtime_fixture /tmp/out @<path>/fixture.args
|
||||||
|
|
||||||
The consumer keeps its own `.args` file in its repository and runs the same command with the
|
The consumer keeps its own `.args` file in its repository and runs the same command with the
|
||||||
same machine-id bind mount as above. Global flags: `--source-repository`, `--detail`. Each
|
same machine-id bind mount as above. Global flags: `--source-repository`, `--detail`. Each
|
||||||
|
|
|
||||||
|
|
@ -2795,7 +2795,7 @@ async function buildInteropPeers(): Promise<void> {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function buildRustInteropPeer(): Promise<void> {
|
async function buildRustInteropPeer(): Promise<void> {
|
||||||
rustInteropPeerBuild ??= execFileAsync(cargoCommand, ["build", "--quiet", "--example", "cultnet_interop_peer"], {
|
rustInteropPeerBuild ??= execFileAsync(cargoCommand, ["build", "--locked", "--quiet", "--example", "cultnet_interop_peer"], {
|
||||||
cwd: cultnetRsRoot,
|
cwd: cultnetRsRoot,
|
||||||
}).then(() => undefined);
|
}).then(() => undefined);
|
||||||
await rustInteropPeerBuild;
|
await rustInteropPeerBuild;
|
||||||
|
|
|
||||||
53
packages/cultnet-ts/test/raw-put-stamp.test.ts
Normal file
53
packages/cultnet-ts/test/raw-put-stamp.test.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
// A peer's raw put is stored under the stamp it carries; one that is no stamp is refused and nothing is minted or stored
|
||||||
|
// (variants:ruling:raw-put-without-stamp-refused). The registry is the node-level owner: every row of the shared vector.
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import test from "node:test";
|
||||||
|
import { decode } from "@msgpack/msgpack";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { CultCache, defineDocumentType, encodeDocumentPayload } from "@gamecult/cultcache-ts";
|
||||||
|
import { CultNetDocumentRegistry, defineCultNetDocumentBinding, parseCultNetMessage, type CultNetMessage } from "../src";
|
||||||
|
|
||||||
|
const VECTOR = join(__dirname, "..", "..", "..", "..", "tests", "vectors", "stored-at-stamp", "stamps.msgpack");
|
||||||
|
const note = defineDocumentType({
|
||||||
|
type: "tests.stamped_note",
|
||||||
|
schemaId: "tests.stamped_note.v0",
|
||||||
|
schema: z.object({ noteId: z.string(), text: z.string() }),
|
||||||
|
name: "noteId",
|
||||||
|
});
|
||||||
|
const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition: note })]);
|
||||||
|
|
||||||
|
function putOf(key: string, storedAt: unknown): CultNetMessage {
|
||||||
|
const payload = encodeDocumentPayload(note, { noteId: key, text: "peer" });
|
||||||
|
const document: Record<string, unknown> = { schemaId: note.schemaId, recordKey: key, payloadEncoding: "messagepack", payload };
|
||||||
|
if (storedAt !== undefined) document.storedAt = storedAt;
|
||||||
|
return { schemaVersion: "cultnet.document_put_raw.v0", messageId: `put-${key}`, document } as unknown as CultNetMessage;
|
||||||
|
}
|
||||||
|
|
||||||
|
test("the registry stores a raw put only under a stamp, for every row of the vector and for a missing or mistyped one", async () => {
|
||||||
|
const rows = decode(readFileSync(VECTOR)) as [string, boolean][];
|
||||||
|
const cases: [string, unknown, boolean][] = [...rows.map(([text, parses]): [string, unknown, boolean] => [JSON.stringify(text), text, parses]),
|
||||||
|
["missing", undefined, false], ["number", 1_700_000_000_000, false], ["null", null, false]];
|
||||||
|
const cache = CultCache.builder().withDocumentType(note).build();
|
||||||
|
for (const [label, storedAt, parses] of cases) {
|
||||||
|
const message = putOf("k", storedAt);
|
||||||
|
// The parser lets a put through to its judges: the server and the registry refuse it, naming the document.
|
||||||
|
parseCultNetMessage(message, "cultnet.schema.v0");
|
||||||
|
const before = cache.getEnvelope(note, "k");
|
||||||
|
if (parses) {
|
||||||
|
await registry.applyRawDocumentPutMessage(cache, message as never);
|
||||||
|
assert.equal(cache.getRequiredEnvelope(note, "k").storedAt.length > 0, true, label);
|
||||||
|
} else {
|
||||||
|
await assert.rejects(registry.applyRawDocumentPutMessage(cache, message as never), /storedAt stamp/, label);
|
||||||
|
assert.deepEqual(cache.getEnvelope(note, "k"), before, `${label}: nothing minted or stored`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The refused puts left the store writable.
|
||||||
|
await registry.applyRawDocumentPutMessage(cache, putOf("after", "2026-11-07T00:00:00Z") as never);
|
||||||
|
assert.equal(cache.getRequired(note, "after").text, "peer");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a refused put does not serve either: createRawSnapshotResponseForPut refuses it too", () => {
|
||||||
|
assert.throws(() => registry.createRawSnapshotResponseForPut(putOf("k", "yesterday") as never, "m"), /storedAt stamp/);
|
||||||
|
});
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
//
|
//
|
||||||
// The vectors below were produced by `cultnet-rs`, not written by hand:
|
// The vectors below were produced by `cultnet-rs`, not written by hand:
|
||||||
//
|
//
|
||||||
// cargo run --example runtime_presence_vector -p cultnet-rs
|
// cargo run --locked --example runtime_presence_vector -p cultnet-rs
|
||||||
//
|
//
|
||||||
// That matters. A fixture authored for both sides at once agrees with itself
|
// That matters. A fixture authored for both sides at once agrees with itself
|
||||||
// and proves nothing; the contract is only real if one fork emits the bytes and
|
// and proves nothing; the contract is only real if one fork emits the bytes and
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ plus the rules that need no store to state (`Answer::is_training_data`, `Answer:
|
||||||
(comments 682 and 759). Specification only: no collector, tally, export or tombstone hashing lives here.
|
(comments 682 and 759). Specification only: no collector, tally, export or tombstone hashing lives here.
|
||||||
|
|
||||||
Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs`
|
Entry points: `src/lib.rs`; golden vectors `tests/vectors/consent_golden.csv`, run by `tests/golden.rs`
|
||||||
(`cargo test`). Planned, not live: the collector, the tally, the training export.
|
(`cargo test --locked`). Planned, not live: the collector, the tally, the training export.
|
||||||
|
|
||||||
## Where each ruling lands (doc lines at aaeb175)
|
## Where each ruling lands (doc lines at aaeb175)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,8 @@ adversary, storage, and any projection. Every parameter is a placeholder until t
|
||||||
## First call
|
## First call
|
||||||
|
|
||||||
```
|
```
|
||||||
cargo run --example first_call # prints the Meta example and a splitting table
|
cargo run --locked --example first_call # prints the Meta example and a splitting table
|
||||||
cargo test
|
cargo test --locked
|
||||||
```
|
```
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
|
|
|
||||||
|
|
@ -64,48 +64,7 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
||||||
var reports = new List<CultSchemaMigrationReport>();
|
var reports = new List<CultSchemaMigrationReport>();
|
||||||
var loaded = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
|
var loaded = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
|
||||||
var foreign = new List<CultForeignRecord>();
|
var foreign = new List<CultForeignRecord>();
|
||||||
for (var attempt = 1; ; attempt++)
|
LoadManifestPages(null, loaded, reports, foreign, Trace);
|
||||||
{
|
|
||||||
loaded.Clear();
|
|
||||||
reports.Clear();
|
|
||||||
foreign.Clear();
|
|
||||||
// Only pages named by the manifest read under the lease are loaded; orphaned pages are never loaded. With no
|
|
||||||
// lock to lease (none created yet, or a store copied without it) a writer may commit mid-load, so an unleased
|
|
||||||
// load stands only if its manifest is still current; if the manifest moved it reloads, under the lease the
|
|
||||||
// writer created. A page that fails under an unmoved manifest is corruption, not a race.
|
|
||||||
using var lease = AcquireCommitLease(wait: true, create: false);
|
|
||||||
var manifestBytes = ReadManifestBytes();
|
|
||||||
var (manifest, _) = ParseManifest(manifestBytes);
|
|
||||||
Trace($"manifest records={manifest.Records.Length}");
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (lease == null)
|
|
||||||
UnleasedManifestRead?.Invoke();
|
|
||||||
LoadRecordPages(
|
|
||||||
manifest.Records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(),
|
|
||||||
manifest.SchemaCatalog,
|
|
||||||
loaded,
|
|
||||||
reports,
|
|
||||||
foreign);
|
|
||||||
if (lease != null || !ManifestMoved(manifestBytes))
|
|
||||||
{
|
|
||||||
Trace($"indexed-pages loaded={loaded.Count}");
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception exception) when (lease == null && IsTornGeneration(exception))
|
|
||||||
{
|
|
||||||
// The re-read stays out of the filter: an exception thrown inside a filter is swallowed as false.
|
|
||||||
if (!ManifestMoved(manifestBytes))
|
|
||||||
throw;
|
|
||||||
if (attempt == UnleasedLoadAttempts)
|
|
||||||
throw Unsettled(exception);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (attempt == UnleasedLoadAttempts)
|
|
||||||
throw Unsettled(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool Staged(string key) => _dirtyKeys.ContainsKey(key) || _deletedKeys.ContainsKey(key);
|
bool Staged(string key) => _dirtyKeys.ContainsKey(key) || _deletedKeys.ContainsKey(key);
|
||||||
var arrived = loaded
|
var arrived = loaded
|
||||||
|
|
@ -136,6 +95,70 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
||||||
Trace("publish");
|
Trace("publish");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The one load of manifest pages, for the whole store (key null) or one record: under the lease, with the moved-manifest retry.
|
||||||
|
// It reads only; adopting what it loaded is the caller's.
|
||||||
|
private void LoadManifestPages(
|
||||||
|
string? key,
|
||||||
|
Dictionary<string, CultStoredDocument> loaded,
|
||||||
|
List<CultSchemaMigrationReport> reports,
|
||||||
|
List<CultForeignRecord> foreign,
|
||||||
|
Action<string> trace)
|
||||||
|
{
|
||||||
|
for (var attempt = 1; ; attempt++)
|
||||||
|
{
|
||||||
|
loaded.Clear();
|
||||||
|
reports.Clear();
|
||||||
|
foreign.Clear();
|
||||||
|
// Only pages named by the manifest read under the lease are loaded; orphaned pages are never loaded. With no
|
||||||
|
// lock to lease (none created yet, or a store copied without it) a writer may commit mid-load, so an unleased
|
||||||
|
// load stands only if its manifest is still current; if the manifest moved it reloads, under the lease the
|
||||||
|
// writer created. A page that fails under an unmoved manifest is corruption, not a race.
|
||||||
|
using var lease = AcquireCommitLease(wait: true, create: false);
|
||||||
|
var manifestBytes = ReadManifestBytes();
|
||||||
|
var (manifest, _) = ParseManifest(manifestBytes);
|
||||||
|
trace($"manifest records={manifest.Records.Length}");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (lease == null)
|
||||||
|
UnleasedManifestRead?.Invoke();
|
||||||
|
LoadRecordPages(
|
||||||
|
manifest.Records.Where(record => key == null || string.Equals(record.Key, key, StringComparison.Ordinal))
|
||||||
|
.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray(),
|
||||||
|
manifest.SchemaCatalog,
|
||||||
|
loaded,
|
||||||
|
reports,
|
||||||
|
foreign);
|
||||||
|
if (lease != null || !ManifestMoved(manifestBytes))
|
||||||
|
{
|
||||||
|
trace($"indexed-pages loaded={loaded.Count}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (lease == null && IsTornGeneration(exception))
|
||||||
|
{
|
||||||
|
// The re-read stays out of the filter: an exception thrown inside a filter is swallowed as false.
|
||||||
|
if (!ManifestMoved(manifestBytes))
|
||||||
|
throw;
|
||||||
|
if (attempt == UnleasedLoadAttempts)
|
||||||
|
throw Unsettled(exception);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt == UnleasedLoadAttempts)
|
||||||
|
throw Unsettled(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The record at a key as its page holds it now, decoded fresh and stamped with VersionOf its manifest payload, the version
|
||||||
|
// CommitBatch compares. Null when none is there or it is foreign. Touches no entry, hydration, report or foreign list of the
|
||||||
|
// store, and raises nothing.
|
||||||
|
internal override CultStoredDocument? ReadDurable(string key) => Held(() =>
|
||||||
|
{
|
||||||
|
var loaded = new Dictionary<string, CultStoredDocument>(StringComparer.Ordinal);
|
||||||
|
LoadManifestPages(key, loaded, new List<CultSchemaMigrationReport>(), new List<CultForeignRecord>(), _ => { });
|
||||||
|
return loaded.TryGetValue(key, out var stored) ? stored : null;
|
||||||
|
});
|
||||||
|
|
||||||
// Variants wait for the directory store's own cut; until then it refuses one rather than persist a resolved view as a plain page.
|
// Variants wait for the directory store's own cut; until then it refuses one rather than persist a resolved view as a plain page.
|
||||||
private static void RefuseVariant(CultStoredDocument entry)
|
private static void RefuseVariant(CultStoredDocument entry)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -2052,6 +2052,29 @@ namespace GameCult.Caching
|
||||||
Contended
|
Contended
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public enum CultUpdateOutcome
|
||||||
|
{
|
||||||
|
Updated,
|
||||||
|
Declined,
|
||||||
|
LostRace
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// What <see cref="CultCache.Update{T}"/> did. A lost race is a value, not an exception. <see cref="Document"/> is the instance the
|
||||||
|
/// cache holds at the key when Update returns: the written one for Updated, null when the key is absent.
|
||||||
|
/// </summary>
|
||||||
|
public readonly struct CultUpdateResult<T> where T : class
|
||||||
|
{
|
||||||
|
public CultUpdateResult(CultUpdateOutcome outcome, T? document)
|
||||||
|
{
|
||||||
|
Outcome = outcome;
|
||||||
|
Document = document;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CultUpdateOutcome Outcome { get; }
|
||||||
|
public T? Document { get; }
|
||||||
|
}
|
||||||
|
|
||||||
public sealed class CultCommitRequest
|
public sealed class CultCommitRequest
|
||||||
{
|
{
|
||||||
internal CultCommitRequest(
|
internal CultCommitRequest(
|
||||||
|
|
@ -2563,6 +2586,7 @@ namespace GameCult.Caching
|
||||||
return UpsertAsync(document, handle);
|
return UpsertAsync(document, handle);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A blind write, last-writer-wins per key; Update is the conditional golden path.
|
||||||
// Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares.
|
// Refused with CultSchemaConflictException when the key holds a record stored under an id no registered type declares.
|
||||||
public Task<CultRecordHandle<T>> UpsertAsync<T>(T document, CultRecordHandle<T>? handle = null)
|
public Task<CultRecordHandle<T>> UpsertAsync<T>(T document, CultRecordHandle<T>? handle = null)
|
||||||
{
|
{
|
||||||
|
|
@ -2639,6 +2663,7 @@ namespace GameCult.Caching
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published.
|
/// Commits the staged batch. Returns false when a condition failed: nothing was written, changed in memory, or published.
|
||||||
|
/// Without <c>Expect</c> a commit is blind; <see cref="Update{T}"/> retries a conditional one.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer
|
/// A commit that succeeds and then fails in a journal or observer still stands and is published, and the writer
|
||||||
|
|
@ -2656,6 +2681,93 @@ namespace GameCult.Caching
|
||||||
return Land(stage, wait: false);
|
return Land(stage, wait: false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads the record at <paramref name="key"/> as its home store holds it, applies <paramref name="apply"/> to a fresh copy of it, and
|
||||||
|
/// commits the result only if the record is still the one that was read; when another writer got there first it re-reads and runs
|
||||||
|
/// <paramref name="apply"/> again, up to <paramref name="maxAttempts"/> times. The write is durable when Update returns.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <paramref name="apply"/> may run up to <paramref name="maxAttempts"/> times. It gets the record as the store that holds it has it now, read from every home
|
||||||
|
/// of T and its routed subtypes and decoded afresh for this attempt (null when the key holds nothing), which it may change and return. A record held by two stores, or an apply that changes the record's home store, throws. A change made in place to an
|
||||||
|
/// instance the cache holds is neither seen nor written. It must not write to the cache or have other side effects. Return null to write
|
||||||
|
/// nothing. A store with staged blind writes refuses Update until it is flushed, as it refuses any conditional commit. A cache with no
|
||||||
|
/// store attached copies its held instance and needs a codec; a store kind outside the library cannot serve Update (<see cref="NotSupportedException"/>). Running out of attempts returns <see cref="CultUpdateOutcome.LostRace"/>; every other failure throws.
|
||||||
|
/// </remarks>
|
||||||
|
public CultUpdateResult<T> Update<T>(CultRecordKey key, Func<T?, T?> apply, int maxAttempts = 5) where T : class
|
||||||
|
{
|
||||||
|
if (apply == null) throw new ArgumentNullException(nameof(apply));
|
||||||
|
if (maxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "At least one attempt is required.");
|
||||||
|
for (var attempt = 1; ; attempt++)
|
||||||
|
{
|
||||||
|
var (version, copy, source) = Held(() =>
|
||||||
|
{
|
||||||
|
if (_stores.Count == 0)
|
||||||
|
{
|
||||||
|
// No store: the held entry is the record.
|
||||||
|
_entries.TryGetValue(key.Value, out var held);
|
||||||
|
var codec = _codec ?? throw new InvalidOperationException("This cache has no codec; create it with CultCacheMessagePack.Create.");
|
||||||
|
if (held == null)
|
||||||
|
return ((string?)null, (T?)null, (CacheBackingStore?)null);
|
||||||
|
if (held.Variant != null || held.Document is not T)
|
||||||
|
throw new InvalidOperationException($"Record {key.Value} is a variant or not a {typeof(T).Name}; Update cannot copy it.");
|
||||||
|
return (held.StoredVersion, (T?)codec.Clone(held.Document, held.Descriptor.DocumentType), null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The store with the key is among the homes of T and its routed subtypes; the cache's own view of the key may be stale, so it is not asked.
|
||||||
|
var homes = CandidateHomes(typeof(T));
|
||||||
|
if (homes.Length == 0)
|
||||||
|
throw new InvalidOperationException("A conditional batch names its home store through a record it upserts or removes.");
|
||||||
|
CacheBackingStore? holder = null;
|
||||||
|
CultStoredDocument? durable = null;
|
||||||
|
foreach (var candidate in homes)
|
||||||
|
{
|
||||||
|
if (candidate.ReadDurable(key.Value) is not { } found)
|
||||||
|
continue;
|
||||||
|
if (holder != null)
|
||||||
|
throw new InvalidOperationException($"Record {key.Value} is held by two stores, {holder} and {candidate}; Update cannot choose.");
|
||||||
|
holder = candidate;
|
||||||
|
durable = found;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (durable == null)
|
||||||
|
return ((string?)null, (T?)null, (CacheBackingStore?)null);
|
||||||
|
if (durable.Variant != null || durable.Document is not T typed)
|
||||||
|
throw new InvalidOperationException($"Record {key.Value} is a variant or not a {typeof(T).Name}; Update cannot copy it.");
|
||||||
|
CultElementIds.Plan(durable.Descriptor.DocumentType, durable.Document, key.Value, deterministic: true).Apply();
|
||||||
|
return (durable.StoredVersion, typed, holder);
|
||||||
|
});
|
||||||
|
var next = apply(copy);
|
||||||
|
if (next == null)
|
||||||
|
return new CultUpdateResult<T>(CultUpdateOutcome.Declined, Get<T>(key));
|
||||||
|
if (source != null && Held(() => Home(next.GetType())) is var landing && landing != source)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Update of {key.Value} would move the record from {source} to {landing?.ToString() ?? "memory"}; remove it and write it at its new home.");
|
||||||
|
CultCommitOutcome outcome;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
outcome = Land(batch =>
|
||||||
|
{
|
||||||
|
batch.Expected.Add((key, version));
|
||||||
|
batch.Upsert(next, new CultRecordHandle<T>(key));
|
||||||
|
}, wait: true);
|
||||||
|
}
|
||||||
|
catch (CultWriteConflictException conflict) when (conflict.InnerException == null)
|
||||||
|
{
|
||||||
|
outcome = CultCommitOutcome.Mismatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (outcome == CultCommitOutcome.Committed)
|
||||||
|
return new CultUpdateResult<T>(CultUpdateOutcome.Updated, next);
|
||||||
|
if (outcome != CultCommitOutcome.Mismatch)
|
||||||
|
throw new InvalidOperationException($"Update of {key.Value} was refused: {outcome}.");
|
||||||
|
if (attempt >= maxAttempts)
|
||||||
|
{
|
||||||
|
Held(() => { Home(next.GetType())?.PullAll(); return true; });
|
||||||
|
return new CultUpdateResult<T>(CultUpdateOutcome.LostRace, Get<T>(key));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public CultRecordHandle<T>? TryGetHandle<T>(T document)
|
public CultRecordHandle<T>? TryGetHandle<T>(T document)
|
||||||
{
|
{
|
||||||
if (document == null) throw new ArgumentNullException(nameof(document));
|
if (document == null) throw new ArgumentNullException(nameof(document));
|
||||||
|
|
@ -3673,6 +3785,18 @@ namespace GameCult.Caching
|
||||||
return changes;
|
return changes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Every store a record of this type, or of a registered subtype of it, can live in.
|
||||||
|
private CacheBackingStore[] CandidateHomes(Type type)
|
||||||
|
{
|
||||||
|
var homes = new HashSet<CacheBackingStore>();
|
||||||
|
if (Home(type) is { } own)
|
||||||
|
homes.Add(own);
|
||||||
|
foreach (var descriptor in _registry.AllDescriptors)
|
||||||
|
if (type.IsAssignableFrom(descriptor.DocumentType) && Home(descriptor.DocumentType) is { } routed)
|
||||||
|
homes.Add(routed);
|
||||||
|
return _stores.Select(entry => entry.Store).Where(homes.Contains).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
// The most derived routed home type assignable from the document type, else the untyped store.
|
// The most derived routed home type assignable from the document type, else the untyped store.
|
||||||
private CacheBackingStore? Home(Type documentType)
|
private CacheBackingStore? Home(Type documentType)
|
||||||
{
|
{
|
||||||
|
|
@ -3949,6 +4073,15 @@ namespace GameCult.Caching
|
||||||
internal void UseRegistry(CultDocumentRegistry? registry) => _registry = registry;
|
internal void UseRegistry(CultDocumentRegistry? registry) => _registry = registry;
|
||||||
|
|
||||||
public abstract void PullAll();
|
public abstract void PullAll();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The record at <paramref name="key"/> as this store holds it now, decoded as a load decodes it by this call (a new instance nobody
|
||||||
|
/// else holds) and stamped with the version this store's commit compares. Null when the store holds nothing there or holds it foreign.
|
||||||
|
/// It changes none of the store's entries, staged writes or reports and raises nothing. Internal because only this library can stamp a version.
|
||||||
|
/// </summary>
|
||||||
|
internal virtual CultStoredDocument? ReadDurable(string key) =>
|
||||||
|
throw new NotSupportedException($"{this} cannot read one record, which Update needs.");
|
||||||
|
|
||||||
public abstract void Push(CultStoredDocument entry);
|
public abstract void Push(CultStoredDocument entry);
|
||||||
public abstract void Delete(CultStoredDocument entry);
|
public abstract void Delete(CultStoredDocument entry);
|
||||||
public void PushAll() => PushAll(soft: false);
|
public void PushAll() => PushAll(soft: false);
|
||||||
|
|
@ -5095,6 +5228,9 @@ namespace GameCult.Caching
|
||||||
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializeMarked, out _);
|
return ToStoredDocument(record, durable.Version(key), durable.Snapshot.SchemaCatalog, DeserializeMarked, out _);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update's read: the durable record at the key, decoded afresh, stamped with the version CommitBatch compares.
|
||||||
|
internal override CultStoredDocument? ReadDurable(string key) => Held(() => DurableAt(ReadSnapshot(), key));
|
||||||
|
|
||||||
// The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes:
|
// The file now holds every record written, under the id and at the storedAt the write gave it, and the version of those bytes:
|
||||||
// the store's entries and the cache's say so. A copied record is on disk as the file had it, so nothing learns anything new about it: in particular the
|
// the store's entries and the cache's say so. A copied record is on disk as the file had it, so nothing learns anything new about it: in particular the
|
||||||
// ids it minted at load are still in memory only.
|
// ids it minted at load are still in memory only.
|
||||||
|
|
|
||||||
|
|
@ -352,7 +352,7 @@ namespace GameCult.Caching
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public object Clone(object document, Type type) => _deserialize(type, _serialize(document, type, type));
|
public object Clone(object document, Type type) => _codec.Clone(document, type);
|
||||||
|
|
||||||
public CultInspectorEdit BeginEdit(CultStoredDocument record)
|
public CultInspectorEdit BeginEdit(CultStoredDocument record)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,11 @@ namespace GameCult.Caching
|
||||||
// (document payload, slot and encoded value pairs)
|
// (document payload, slot and encoded value pairs)
|
||||||
public Func<byte[], IReadOnlyList<KeyValuePair<int, byte[]>>, byte[]> Overlay { get; }
|
public Func<byte[], IReadOnlyList<KeyValuePair<int, byte[]>>, byte[]> Overlay { get; }
|
||||||
|
|
||||||
|
// The one copy of a document: its encoding, decoded again. Update's private argument and CultInspectorModel.Clone share it.
|
||||||
|
internal object Clone(object document, Type type) =>
|
||||||
|
Deserialize(type, type, Serialize(document, type, type))
|
||||||
|
?? throw new InvalidOperationException($"The codec returned null for {type.FullName}.");
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Decodes a document payload read from a store file, as a cache load decodes it: the ids a load mints are minted from the key
|
/// Decodes a document payload read from a store file, as a cache load decodes it: the ids a load mints are minted from the key
|
||||||
/// and their path, so every reader of one store mints the same ids, and a payload a cache write would refuse is refused here too.
|
/// and their path, so every reader of one store mints the same ids, and a payload a cache write would refuse is refused here too.
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ Cacheable models are plain classes annotated with cache intent.
|
||||||
using GameCult.Caching;
|
using GameCult.Caching;
|
||||||
using MessagePack;
|
using MessagePack;
|
||||||
|
|
||||||
|
[MessagePackObject]
|
||||||
[CultDocument("gamecult.item_data", "gamecult.item_data.v1")]
|
[CultDocument("gamecult.item_data", "gamecult.item_data.v1")]
|
||||||
public sealed class ItemData
|
public sealed class ItemData
|
||||||
{
|
{
|
||||||
|
|
@ -131,18 +132,33 @@ using GameCult.Caching;
|
||||||
using GameCult.Caching.MessagePack;
|
using GameCult.Caching.MessagePack;
|
||||||
|
|
||||||
var cache = await CultCacheMessagePack.OpenAsync("Data.msgpack");
|
var cache = await CultCacheMessagePack.OpenAsync("Data.msgpack");
|
||||||
|
var key = new CultRecordKey("potion");
|
||||||
|
|
||||||
|
// The golden path: read, change a private copy, commit only if the record is unchanged, re-run on a lost race.
|
||||||
|
// Durable when it returns; no flush.
|
||||||
|
var result = cache.Update<ItemData>(key, item =>
|
||||||
|
{
|
||||||
|
item ??= new ItemData { Name = "Potion", Sku = "POT-1", Category = "Consumable" };
|
||||||
|
item.Value += 10;
|
||||||
|
return item;
|
||||||
|
});
|
||||||
|
|
||||||
|
// The blind opt-out: last writer wins per key. Name the key, or a new key claims the same unique Sku and is refused.
|
||||||
var handle = await cache.UpsertAsync(new ItemData
|
var handle = await cache.UpsertAsync(new ItemData
|
||||||
{
|
{
|
||||||
Name = "Potion",
|
Name = "Potion",
|
||||||
Sku = "POT-1",
|
Sku = "POT-1",
|
||||||
Category = "Consumable",
|
Category = "Consumable",
|
||||||
Value = 50
|
Value = 50
|
||||||
});
|
}, new CultRecordHandle<ItemData>(key));
|
||||||
|
|
||||||
await cache.FlushAsync();
|
await cache.FlushAsync();
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`Update` may run your function more than once, so it must have no side effects. Returning null writes nothing, and a
|
||||||
|
lost race that outlasts `maxAttempts` comes back as `CultUpdateOutcome.LostRace`. `UpsertAsync` plus `FlushAsync` never
|
||||||
|
checks what it overwrites.
|
||||||
|
|
||||||
If you want the explicit assembly rite, the lower-level `CultCache` +
|
If you want the explicit assembly rite, the lower-level `CultCache` +
|
||||||
`SingleFileMessagePackBackingStore` path is still there. The helper is just the
|
`SingleFileMessagePackBackingStore` path is still there. The helper is just the
|
||||||
blessed front door.
|
blessed front door.
|
||||||
|
|
@ -294,13 +310,10 @@ Useful surfaces:
|
||||||
- `cache.IsDirty`
|
- `cache.IsDirty`
|
||||||
- `store.IsDirty`
|
- `store.IsDirty`
|
||||||
- `cache.Document<T>(key)`
|
- `cache.Document<T>(key)`
|
||||||
- `cache.Soa<T>()`
|
- `cache.Update<T>(key, apply)`
|
||||||
- `cache.UpsertAsync(document)`
|
- `cache.UpsertAsync(document)` (blind write)
|
||||||
- `cache.FlushAllBackingStores()`
|
- `cache.FlushAllBackingStores()`
|
||||||
- `cache.FlushAsync()`
|
- `cache.FlushAsync()`
|
||||||
- `cache.FlushBackingStore(store)`
|
|
||||||
- `cache.PrepareForReloadOrShutdown()`
|
|
||||||
- `cache.PrepareForReloadOrShutdownAsync()`
|
|
||||||
- `cache.FlushAttachedStoresOnDispose`
|
- `cache.FlushAttachedStoresOnDispose`
|
||||||
- `store.FlushOnDispose`
|
- `store.FlushOnDispose`
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1055,6 +1055,13 @@ namespace GameCult.Networking
|
||||||
var key = new CultRecordKey(message.Document.RecordKey);
|
var key = new CultRecordKey(message.Document.RecordKey);
|
||||||
var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document);
|
var descriptor = _documents.ResolveDescriptorForRawDocument(message.Document);
|
||||||
authorize?.Invoke(descriptor.SchemaId, key);
|
authorize?.Invoke(descriptor.SchemaId, key);
|
||||||
|
// A peer's put is judged for its stamp after its authority and before any shard is asked (so it is never forwarded):
|
||||||
|
// the cache would mint a stamp for one that is no stamp, and a refused put stores and mints nothing.
|
||||||
|
if (authorize != null && !CultNetStoredAtStamp.IsStamp(message.Document.StoredAt))
|
||||||
|
{
|
||||||
|
throw new CultNetStoredAtStampException();
|
||||||
|
}
|
||||||
|
|
||||||
var shard = ResolveShardInternal(descriptor, key);
|
var shard = ResolveShardInternal(descriptor, key);
|
||||||
EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch);
|
EnsurePrimary(shard, descriptor.SchemaId, key, message.ShardEpoch);
|
||||||
var document = _documents.DeserializeRawDocument(message.Document);
|
var document = _documents.DeserializeRawDocument(message.Document);
|
||||||
|
|
|
||||||
|
|
@ -475,6 +475,11 @@ namespace GameCult.Networking
|
||||||
_server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}");
|
_server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}");
|
||||||
Refuse(peer, write, CultNetErrorMessage.ForRemoteWriteRefused(ex.Request, ex.Reason));
|
Refuse(peer, write, CultNetErrorMessage.ForRemoteWriteRefused(ex.Request, ex.Reason));
|
||||||
}
|
}
|
||||||
|
catch (CultNetStoredAtStampException ex)
|
||||||
|
{
|
||||||
|
_server.Logger.LogWarning($"CultNet raw put refused: {ex.Message}");
|
||||||
|
Refuse(peer, write, CultNetErrorMessage.ForMessageInvalid("document", "document.storedAt is not a stamp"));
|
||||||
|
}
|
||||||
catch (CultNetShardAuthorityException ex)
|
catch (CultNetShardAuthorityException ex)
|
||||||
{
|
{
|
||||||
bool forwarded;
|
bool forwarded;
|
||||||
|
|
|
||||||
63
src/GameCult.Networking/CultNetStoredAtStamp.cs
Normal file
63
src/GameCult.Networking/CultNetStoredAtStamp.cs
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace GameCult.Networking
|
||||||
|
{
|
||||||
|
// The one shape a peer's storedAt must have: RFC 3339 date-time, YYYY-MM-DDThh:mm:ss, an optional fraction of 1 to 9 digits,
|
||||||
|
// then Z or +hh:mm / -hh:mm. Uppercase T and Z, ASCII digits, no whitespace, a real calendar date, seconds 00-59, years
|
||||||
|
// 0001-9999 for both the local date and the UTC instant, offset hours 00-23 and minutes 00-59. Checked by shape and range, never by DateTimeOffset.TryParse, which is
|
||||||
|
// culture-aware and takes far more than this (tests/vectors/stored-at-stamp, variants:ruling:raw-put-without-stamp-refused).
|
||||||
|
internal static class CultNetStoredAtStamp
|
||||||
|
{
|
||||||
|
public static bool IsStamp(string? text)
|
||||||
|
{
|
||||||
|
if (text == null || text.Length < 20) return false;
|
||||||
|
if (!Digits(text, 0, 4, out var year) || text[4] != '-' || !Digits(text, 5, 2, out var month) || text[7] != '-'
|
||||||
|
|| !Digits(text, 8, 2, out var day) || text[10] != 'T' || !Digits(text, 11, 2, out var hour) || text[13] != ':'
|
||||||
|
|| !Digits(text, 14, 2, out var minute) || text[16] != ':' || !Digits(text, 17, 2, out var second))
|
||||||
|
return false;
|
||||||
|
if (year < 1 || month is < 1 or > 12 || day < 1 || day > DateTime.DaysInMonth(year, month)
|
||||||
|
|| hour > 23 || minute > 59 || second > 59)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
var i = 19;
|
||||||
|
if (text[i] == '.')
|
||||||
|
{
|
||||||
|
var start = ++i;
|
||||||
|
while (i < text.Length && IsDigit(text[i])) i++;
|
||||||
|
if (i - start is < 1 or > 9) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i >= text.Length) return false;
|
||||||
|
if (text[i] == 'Z') return i == text.Length - 1;
|
||||||
|
if (!(text.Length - i == 6 && (text[i] == '+' || text[i] == '-')
|
||||||
|
&& Digits(text, i + 1, 2, out var offsetHour) && text[i + 3] == ':' && Digits(text, i + 4, 2, out var offsetMinute)
|
||||||
|
&& offsetHour <= 23 && offsetMinute <= 59))
|
||||||
|
return false;
|
||||||
|
// The UTC instant must lie in years 0001-9999 too: local minus offset may not leave DateTime's range.
|
||||||
|
var offsetTicks = (offsetHour * 60L + offsetMinute) * TimeSpan.TicksPerMinute * (text[i] == '-' ? -1 : 1);
|
||||||
|
var utcTicks = new DateTime(year, month, day, hour, minute, second).Ticks - offsetTicks;
|
||||||
|
return utcTicks >= DateTime.MinValue.Ticks && utcTicks <= DateTime.MaxValue.Ticks;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsDigit(char c) => c is >= '0' and <= '9';
|
||||||
|
|
||||||
|
private static bool Digits(string text, int start, int count, out int value)
|
||||||
|
{
|
||||||
|
value = 0;
|
||||||
|
for (var i = start; i < start + count; i++)
|
||||||
|
{
|
||||||
|
if (i >= text.Length || !IsDigit(text[i])) return false;
|
||||||
|
value = value * 10 + (text[i] - '0');
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A peer's raw put whose storedAt is no stamp: refused message_invalid, field document, nothing stored or minted.
|
||||||
|
internal sealed class CultNetStoredAtStampException : Exception
|
||||||
|
{
|
||||||
|
public CultNetStoredAtStampException() : base("a raw put document must carry a storedAt stamp") { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,131 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
using MessagePack;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.OutsideStoreTests
|
||||||
|
{
|
||||||
|
// variants:follow_up:update-copy-from-stored-bytes: Update takes its version and its copy from one durable read of the home
|
||||||
|
// store, never from the held instance. Consumer side: public API only, so a change made in place to an instance the cache
|
||||||
|
// hands out can never be committed as if it had been read.
|
||||||
|
public class HeldInstanceUpdateTests
|
||||||
|
{
|
||||||
|
[CultDocument("tests.held_counter", "tests.held_counter.v1")]
|
||||||
|
[MessagePackObject]
|
||||||
|
public sealed class HeldCounter
|
||||||
|
{
|
||||||
|
[Key(0)] [CultName] public string Name { get; set; } = string.Empty;
|
||||||
|
[Key(1)] public int Value { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { typeof(HeldCounter) });
|
||||||
|
private static readonly CultRecordKey Key = new("counter");
|
||||||
|
|
||||||
|
private string _directory = string.Empty;
|
||||||
|
|
||||||
|
[SetUp]
|
||||||
|
public void CreateDirectory()
|
||||||
|
{
|
||||||
|
_directory = Path.Combine(Path.GetTempPath(), $"cultlib-held-update-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(_directory);
|
||||||
|
}
|
||||||
|
|
||||||
|
[TearDown]
|
||||||
|
public void DeleteDirectory()
|
||||||
|
{
|
||||||
|
if (Directory.Exists(_directory))
|
||||||
|
Directory.Delete(_directory, recursive: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CultCache Open(string path, bool directory)
|
||||||
|
{
|
||||||
|
var cache = new CultCache(Registry, CultCacheMessagePack.CreateCodec(Registry));
|
||||||
|
cache.AddBackingStore(directory ? new DirectoryMessagePackBackingStore(path) : new SingleFileMessagePackBackingStore(path));
|
||||||
|
cache.PullAllBackingStoresAsync().GetAwaiter().GetResult();
|
||||||
|
return cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
private string Seed(bool directory, int value)
|
||||||
|
{
|
||||||
|
var path = Path.Combine(_directory, "held.cc");
|
||||||
|
using var cache = Open(path, directory);
|
||||||
|
Assert.That(cache.Commit(batch => batch.Upsert(new HeldCounter { Name = "counter", Value = value }, new CultRecordHandle<HeldCounter>(Key))), Is.True);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int StoredValue(string path, bool directory)
|
||||||
|
{
|
||||||
|
using var reader = Open(path, directory);
|
||||||
|
return reader.Get<HeldCounter>(Key)!.Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static HeldCounter Plus(HeldCounter? counter, int by)
|
||||||
|
{
|
||||||
|
counter!.Value += by;
|
||||||
|
return counter;
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestCase(false)]
|
||||||
|
[TestCase(true)]
|
||||||
|
public void InPlaceChangeToHeldInstanceIsNeitherAppliedNorPersisted(bool directory)
|
||||||
|
{
|
||||||
|
var path = Seed(directory, 1);
|
||||||
|
using var cache = Open(path, directory);
|
||||||
|
cache.Get<HeldCounter>(Key)!.Value = 999;
|
||||||
|
var seen = new List<int>();
|
||||||
|
|
||||||
|
var result = cache.Update<HeldCounter>(Key, counter => { seen.Add(counter!.Value); return Plus(counter, 1); });
|
||||||
|
|
||||||
|
Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated));
|
||||||
|
Assert.That(seen, Is.EqualTo(new[] { 1 }), "apply saw the in-place change");
|
||||||
|
Assert.That(result.Document!.Value, Is.EqualTo(2));
|
||||||
|
Assert.That(StoredValue(path, directory), Is.EqualTo(2), "the in-place 999 was persisted");
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestCase(false)]
|
||||||
|
[TestCase(true)]
|
||||||
|
public void InPlaceChangeToAPreviousResultIsNotCommitted(bool directory)
|
||||||
|
{
|
||||||
|
var path = Seed(directory, 1);
|
||||||
|
using var cache = Open(path, directory);
|
||||||
|
var first = cache.Update<HeldCounter>(Key, counter => Plus(counter, 1));
|
||||||
|
Assert.That(first.Outcome, Is.EqualTo(CultUpdateOutcome.Updated));
|
||||||
|
first.Document!.Value = 999;
|
||||||
|
var seen = new List<int>();
|
||||||
|
|
||||||
|
var second = cache.Update<HeldCounter>(Key, counter => { seen.Add(counter!.Value); return Plus(counter, 1); });
|
||||||
|
|
||||||
|
Assert.That(second.Outcome, Is.EqualTo(CultUpdateOutcome.Updated));
|
||||||
|
Assert.That(seen, Is.EqualTo(new[] { 2 }), "apply ran more than once or saw the in-place change");
|
||||||
|
Assert.That(StoredValue(path, directory), Is.EqualTo(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
[TestCase(false)]
|
||||||
|
[TestCase(true)]
|
||||||
|
public void DeclineAfterInPlaceChangeWritesNothing(bool directory)
|
||||||
|
{
|
||||||
|
var path = Seed(directory, 1);
|
||||||
|
using var cache = Open(path, directory);
|
||||||
|
cache.Get<HeldCounter>(Key)!.Value = 999;
|
||||||
|
var before = Snapshot(path);
|
||||||
|
|
||||||
|
Assert.That(cache.Update<HeldCounter>(Key, _ => null).Outcome, Is.EqualTo(CultUpdateOutcome.Declined));
|
||||||
|
|
||||||
|
Assert.That(Snapshot(path), Is.EqualTo(before));
|
||||||
|
Assert.That(StoredValue(path, directory), Is.EqualTo(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every file of the store: the single file, or the manifest and its pages.
|
||||||
|
private static Dictionary<string, byte[]> Snapshot(string path)
|
||||||
|
{
|
||||||
|
var files = new Dictionary<string, byte[]> { [path] = File.ReadAllBytes(path) };
|
||||||
|
var pages = path + ".records";
|
||||||
|
if (Directory.Exists(pages))
|
||||||
|
foreach (var file in Directory.GetFiles(pages, "*", SearchOption.AllDirectories))
|
||||||
|
files[file] = File.ReadAllBytes(file);
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -364,5 +364,18 @@ namespace GameCult.Caching.OutsideStoreTests
|
||||||
Assert.That(() => cache.Commit(batch => batch.Remove(K)), Throws.InstanceOf<CultSchemaConflictException>(), "removal");
|
Assert.That(() => cache.Commit(batch => batch.Remove(K)), Throws.InstanceOf<CultSchemaConflictException>(), "removal");
|
||||||
Assert.That(disk.Records["k"], Is.EqualTo(before));
|
Assert.That(disk.Records["k"], Is.EqualTo(before));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only the library can stamp a version on a decoded record, so a store kind outside it cannot read one honestly: Update refuses before apply.
|
||||||
|
[Test]
|
||||||
|
public void UpdateOverAnOutsideStoreIsNotSupported()
|
||||||
|
{
|
||||||
|
var disk = new Disk();
|
||||||
|
PutOnDisk(disk, AB, "k", new OutA { Name = "k", Value = 1 });
|
||||||
|
using var cache = Open(disk, AB);
|
||||||
|
var calls = 0;
|
||||||
|
|
||||||
|
Assert.That(() => cache.Update<OutA>(K, a => { calls++; return a; }), Throws.TypeOf<NotSupportedException>());
|
||||||
|
Assert.That(calls, Is.EqualTo(0), "apply ran for a store that cannot serve Update");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
115
tests/GameCult.Caching.Tests/ContentVersionVectorTests.cs
Normal file
115
tests/GameCult.Caching.Tests/ContentVersionVectorTests.cs
Normal file
|
|
@ -0,0 +1,115 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.Buffers;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using GameCult.Caching;
|
||||||
|
using GameCult.Caching.MessagePack;
|
||||||
|
using MessagePack;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.Tests
|
||||||
|
{
|
||||||
|
// tests/vectors/content-version: the content version of a stored record, the lowercase hex SHA-256 of its persisted encoding
|
||||||
|
// (variants:ruling:write-set-content-version-declared-ownership), as the reference reads it from a single-file store. The C#
|
||||||
|
// reference is the only writer: under CULTCACHE_WRITE_VECTORS=1 the store is written and the versions read from it are written
|
||||||
|
// beside it; every other run opens the committed store.cc and compares what the reference reads to versions.msgpack. Another
|
||||||
|
// runtime computes the version from the same file and must equal these rows.
|
||||||
|
public class ContentVersionVectorTests
|
||||||
|
{
|
||||||
|
private const string Vector = "content-version";
|
||||||
|
|
||||||
|
private static readonly Type Item =
|
||||||
|
EmittedDocumentTypes.Emit("CvItem", "tests.cv_item", "tests.cv_item.v1", new[] { new EmittedDocumentTypes.Field("Text", typeof(string), 0) });
|
||||||
|
|
||||||
|
private static readonly CultDocumentRegistry Registry = CultDocumentRegistry.ForTypes(new[] { Item });
|
||||||
|
|
||||||
|
private static readonly (string Key, string Text)[] Records = { ("alpha", "first"), ("beta", "second") };
|
||||||
|
|
||||||
|
private static byte[] StoreBytes()
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-cvv-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.Combine(directory, "store.cc");
|
||||||
|
using (var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry, StoreFlushOnDispose = true }))
|
||||||
|
{
|
||||||
|
cache.Clock = () => VectorFiles.StoredAt;
|
||||||
|
foreach (var (key, text) in Records)
|
||||||
|
cache.UpsertAsync(Item, EmittedDocumentTypes.New(Item, ("Text", text)), new CultRecordKey(key)).GetAwaiter().GetResult();
|
||||||
|
cache.FlushAsync().GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
return File.ReadAllBytes(path);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the reference reads from a store file: [key, version] for each record, sorted by key.
|
||||||
|
private static (string Key, string Version)[] ReadVersions(byte[] store)
|
||||||
|
{
|
||||||
|
var directory = Path.Combine(Path.GetTempPath(), $"cultlib-cvr-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var path = Path.Combine(directory, "store.cc");
|
||||||
|
File.WriteAllBytes(path, store);
|
||||||
|
using var cache = CultCacheMessagePack.Create(path, new CultCacheOpenOptions { Registry = Registry });
|
||||||
|
return cache.AllStoredDocuments
|
||||||
|
.Select(stored => (Key: stored.Key.Value, Version: stored.StoredVersion))
|
||||||
|
.OrderBy(row => row.Key, StringComparer.Ordinal)
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Directory.Delete(directory, recursive: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] RowsFile((string Key, string Version)[] rows)
|
||||||
|
{
|
||||||
|
var buffer = new ArrayBufferWriter<byte>();
|
||||||
|
var writer = new MessagePackWriter(buffer);
|
||||||
|
writer.WriteArrayHeader(rows.Length);
|
||||||
|
foreach (var (key, version) in rows)
|
||||||
|
{
|
||||||
|
writer.WriteArrayHeader(2);
|
||||||
|
writer.Write(key);
|
||||||
|
writer.Write(version);
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.Flush();
|
||||||
|
return buffer.WrittenSpan.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TheReferenceReadsTheCommittedVersions()
|
||||||
|
{
|
||||||
|
if (VectorFiles.Writing)
|
||||||
|
{
|
||||||
|
var store = StoreBytes();
|
||||||
|
VectorFiles.Check(Vector, "store.cc", store);
|
||||||
|
VectorFiles.Check(Vector, "versions.msgpack", RowsFile(ReadVersions(store)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var committed = VectorFiles.Committed(Vector, "store.cc");
|
||||||
|
var rows = ReadVersions(committed);
|
||||||
|
Assert.That(rows.Select(row => row.Key), Is.EqualTo(Records.Select(record => record.Key)), "the store holds the two records");
|
||||||
|
Assert.That(rows.Select(row => row.Version).Distinct().Count(), Is.EqualTo(rows.Length), "two different records, two versions");
|
||||||
|
Assert.That(rows.All(row => row.Version.Length == 64 && row.Version == row.Version.ToLowerInvariant()), Is.True, "lowercase hex SHA-256");
|
||||||
|
Assert.That(Convert.ToHexString(RowsFile(rows)), Is.EqualTo(Convert.ToHexString(VectorFiles.Committed(Vector, "versions.msgpack"))), "versions.msgpack");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public void TheCommittedStoreIsWhatTheReferenceWrites()
|
||||||
|
{
|
||||||
|
if (VectorFiles.Writing) return;
|
||||||
|
Assert.That(Convert.ToHexString(StoreBytes()), Is.EqualTo(Convert.ToHexString(VectorFiles.Committed(Vector, "store.cc"))), "store.cc");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -224,6 +224,28 @@ namespace GameCult.Caching.Tests
|
||||||
Assert.That(AllIds(reread), Is.EqualTo(first), "the ids minted at load are the ids written, and stay through repeated writes");
|
Assert.That(AllIds(reread), Is.EqualTo(first), "the ids minted at load are the ids written, and stay through repeated writes");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update reads the store's bytes, which hold no ids; it mints the ids the load minted, so the write keeps the ids the cache showed.
|
||||||
|
[Test]
|
||||||
|
public void UpdateOfAPreIdRecordPersistsTheIdsTheLoadMinted()
|
||||||
|
{
|
||||||
|
var path = PathOf("old.cc");
|
||||||
|
WritePreIdStore(path, null, ("a", "a", 2));
|
||||||
|
var key = new CultRecordKey("a");
|
||||||
|
|
||||||
|
string[] shown;
|
||||||
|
using (var cache = Open(path))
|
||||||
|
{
|
||||||
|
shown = AllIds(cache.Get<IdDeck>(key)!);
|
||||||
|
Assert.That(shown, Has.All.Not.Empty);
|
||||||
|
var result = cache.Update<IdDeck>(key, deck => { deck!.Name = "a2"; return deck; });
|
||||||
|
Assert.That(result.Outcome, Is.EqualTo(CultUpdateOutcome.Updated));
|
||||||
|
}
|
||||||
|
|
||||||
|
var written = MessagePackSerializer.Deserialize<IdDeck>(DiskRecord(path, "a").Payload);
|
||||||
|
Assert.That(written.Name, Is.EqualTo("a2"));
|
||||||
|
Assert.That(AllIds(written), Is.EqualTo(shown), "the write replaced the ids the cache showed with random ones");
|
||||||
|
}
|
||||||
|
|
||||||
[Test]
|
[Test]
|
||||||
public void MintElementIdsRewritesEveryPreIdRecordAndNothingElse()
|
public void MintElementIdsRewritesEveryPreIdRecordAndNothingElse()
|
||||||
{
|
{
|
||||||
|
|
|
||||||
151
tests/GameCult.Caching.Tests/ElementIdWalkTests.cs
Normal file
151
tests/GameCult.Caching.Tests/ElementIdWalkTests.cs
Normal file
|
|
@ -0,0 +1,151 @@
|
||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Runtime.Serialization;
|
||||||
|
using GameCult.Caching;
|
||||||
|
using MessagePack;
|
||||||
|
using NUnit.Framework;
|
||||||
|
|
||||||
|
namespace GameCult.Caching.Tests
|
||||||
|
{
|
||||||
|
// The id walk's own contracts, one mutant-sized fact at a time: the order of the id path, a union that names itself, a list of
|
||||||
|
// values never counting as an id, and the attribute predicates that decide whether a type is walked or written whole.
|
||||||
|
public class ElementIdWalkTests
|
||||||
|
{
|
||||||
|
// The codec asks MessagePack about the id path in order and refuses at the first type it cannot allow, so the order is what
|
||||||
|
// an error names first: the document, then the collection types from the outermost in, then the element and its arms.
|
||||||
|
[Test]
|
||||||
|
public void TheIdPathListsCollectionsFromTheOutermostIn()
|
||||||
|
{
|
||||||
|
Assert.That(CultElementIds.Inspect(typeof(MsMapMap)).IdPath, Is.EqualTo(new[]
|
||||||
|
{
|
||||||
|
typeof(MsMapMap), typeof(Dictionary<string, Dictionary<string, List<ElementIdEngineTests.EIded>>>),
|
||||||
|
typeof(Dictionary<string, List<ElementIdEngineTests.EIded>>), typeof(List<ElementIdEngineTests.EIded>), typeof(ElementIdEngineTests.EIded)
|
||||||
|
}), "maps nested in maps");
|
||||||
|
Assert.That(CultElementIds.Inspect(typeof(MsListDeep)).IdPath, Is.EqualTo(new[]
|
||||||
|
{
|
||||||
|
typeof(MsListDeep), typeof(List<List<List<ElementIdEngineTests.EIded>>>), typeof(List<List<ElementIdEngineTests.EIded>>),
|
||||||
|
typeof(List<ElementIdEngineTests.EIded>), typeof(ElementIdEngineTests.EIded)
|
||||||
|
}), "lists nested in lists");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConcreteTypes stops at a type it has already expanded, so a union that lists its own base is finished, not a stack overflow.
|
||||||
|
[Test]
|
||||||
|
public void AUnionThatNamesItselfIsWalkedOnce()
|
||||||
|
{
|
||||||
|
var survey = CultElementIds.Inspect(typeof(MsSelfHolder));
|
||||||
|
Assert.That(survey.Problems, Is.Empty);
|
||||||
|
Assert.That(survey.IdPath, Is.EquivalentTo(new[] { typeof(MsSelfHolder), typeof(List<MsSelfBase>), typeof(MsSelfBase), typeof(MsSelfArm) }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Value lists hold no ids however they are nested, and a root that is itself a value list holds none either.
|
||||||
|
[Test]
|
||||||
|
public void ListsOfValuesHoldNoIds()
|
||||||
|
{
|
||||||
|
var values = new MsValues { Numbers = { 1 }, Tags = { "a" }, Grid = { new List<int> { 1 } }, Map = { ["k"] = new List<int> { 2 } } };
|
||||||
|
Assert.That(CultElementIds.Holds(typeof(MsValues), values), Is.False);
|
||||||
|
Assert.That(CultElementIds.Holds(typeof(List<int>), new List<int> { 1, 2 }), Is.False);
|
||||||
|
Assert.That(CultElementIds.Holds(typeof(List<List<int>>), new List<List<int>> { new() { 1 } }), Is.False);
|
||||||
|
Assert.That(CultElementIds.Holds(typeof(List<string>), new List<string> { "a" }), Is.False);
|
||||||
|
Assert.That(CultElementIds.Holds(typeof(MsBoom), new MsBoom()), Is.False, "a sequence of values is not enumerated");
|
||||||
|
}
|
||||||
|
|
||||||
|
// A nullable struct is walked as the struct it wraps; a nullable value is not walked at all.
|
||||||
|
[Test]
|
||||||
|
public void ANullableKeyedStructWalksAsItsOwner()
|
||||||
|
{
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsKeyedStruct)), Is.True);
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsKeyedStruct?)), Is.True);
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(int?)), Is.False);
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(int)), Is.False);
|
||||||
|
}
|
||||||
|
|
||||||
|
// [Key] is read through an override, so an element whose id property overrides a keyed base member is still an id with a key.
|
||||||
|
[Test]
|
||||||
|
public void AKeyOnTheBaseOfAnOverrideCountsForTheOverride()
|
||||||
|
{
|
||||||
|
Assert.That(CultElementIds.Inspect(typeof(MsOverrideHolder)).Problems, Is.Empty);
|
||||||
|
Assert.That(CultElementIds.Inspect(typeof(MsOverrideHolder)).IdPath, Does.Contain(typeof(MsOverrideEl)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// A type is walked by key when it is [MessagePackObject] or [Union], read through its bases for the former; it is one whole
|
||||||
|
// value when it declares its own formatter, is [DataContract] without being [MessagePackObject], or is keyed by name.
|
||||||
|
[Test]
|
||||||
|
public void KeyedTypesWalkAndWholeTypesDoNot()
|
||||||
|
{
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsMpBase)), Is.True, "[MessagePackObject]");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsUBase)), Is.True, "[Union]");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsMpDerived)), Is.True, "[MessagePackObject] inherited");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsPlain)), Is.False, "[Key] members without [MessagePackObject] or [Union]");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsUArm)), Is.True, "an arm that is itself [MessagePackObject]");
|
||||||
|
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsDc)), Is.False, "[DataContract] alone");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsDcMp)), Is.True, "[DataContract] beside [MessagePackObject]");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsDcOverMpBase)), Is.True, "[DataContract] over an inherited [MessagePackObject]");
|
||||||
|
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsPropName)), Is.False, "KeyAsPropertyName");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsPropNameDerived)), Is.False, "KeyAsPropertyName inherited");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsStringKey)), Is.False, "a string [Key]");
|
||||||
|
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsFormattedBase)), Is.False, "[MessagePackFormatter]");
|
||||||
|
Assert.That(CultElementIds.WalksAsOwner(typeof(MsFormattedDerived)), Is.False, "[MessagePackFormatter] inherited");
|
||||||
|
}
|
||||||
|
|
||||||
|
[MessagePackObject] public class MsMapMap { [Key(0)] public Dictionary<string, Dictionary<string, List<ElementIdEngineTests.EIded>>> M { get; set; } = new(); }
|
||||||
|
[MessagePackObject] public class MsListDeep { [Key(0)] public List<List<List<ElementIdEngineTests.EIded>>> G { get; set; } = new(); }
|
||||||
|
|
||||||
|
[Union(0, typeof(MsSelfBase)), Union(1, typeof(MsSelfArm))] public abstract class MsSelfBase { }
|
||||||
|
[MessagePackObject] public class MsSelfArm : MsSelfBase { [Key(0)] [CultElementId] public string Id { get; set; } = ""; }
|
||||||
|
[MessagePackObject] public class MsSelfHolder { [Key(0)] public List<MsSelfBase> Items { get; set; } = new(); }
|
||||||
|
|
||||||
|
[MessagePackObject]
|
||||||
|
public class MsValues
|
||||||
|
{
|
||||||
|
[Key(0)] public List<int> Numbers { get; set; } = new();
|
||||||
|
[Key(1)] public List<string> Tags { get; set; } = new();
|
||||||
|
[Key(2)] public List<List<int>> Grid { get; set; } = new();
|
||||||
|
[Key(3)] public Dictionary<string, List<int>> Map { get; set; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class MsBoom : IEnumerable<int>
|
||||||
|
{
|
||||||
|
public IEnumerator<int> GetEnumerator() => throw new InvalidOperationException("enumerated");
|
||||||
|
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
|
||||||
|
}
|
||||||
|
|
||||||
|
[MessagePackObject] public struct MsKeyedStruct { [Key(0)] public int X; }
|
||||||
|
|
||||||
|
[MessagePackObject]
|
||||||
|
public class MsOverrideBase
|
||||||
|
{
|
||||||
|
[Key(0)] [CultElementId] public virtual string Id { get; set; } = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
[MessagePackObject]
|
||||||
|
public class MsOverrideEl : MsOverrideBase
|
||||||
|
{
|
||||||
|
public override string Id { get; set; } = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
[MessagePackObject] public class MsOverrideHolder { [Key(0)] public List<MsOverrideEl> Items { get; set; } = new(); }
|
||||||
|
|
||||||
|
[MessagePackObject] public class MsMpBase { [Key(0)] public int A { get; set; } }
|
||||||
|
public class MsMpDerived : MsMpBase { }
|
||||||
|
public class MsPlain { [Key(0)] public int A { get; set; } }
|
||||||
|
[Union(0, typeof(MsUArm))] public abstract class MsUBase { }
|
||||||
|
[MessagePackObject] public class MsUArm : MsUBase { [Key(0)] public int A { get; set; } }
|
||||||
|
|
||||||
|
[DataContract] public class MsDc { [DataMember] public int A { get; set; } }
|
||||||
|
[DataContract] [MessagePackObject] public class MsDcMp { [Key(0)] public int A { get; set; } }
|
||||||
|
[DataContract] public class MsDcOverMpBase : MsMpBase { }
|
||||||
|
|
||||||
|
[MessagePackObject(keyAsPropertyName: true)] public class MsPropName { public int A { get; set; } }
|
||||||
|
public class MsPropNameDerived : MsPropName { }
|
||||||
|
[MessagePackObject] public class MsStringKey { [Key("a")] public int A { get; set; } }
|
||||||
|
|
||||||
|
public sealed class MsNoFormatter { }
|
||||||
|
[MessagePackFormatter(typeof(MsNoFormatter))] [MessagePackObject] public class MsFormattedBase { [Key(0)] public int A { get; set; } }
|
||||||
|
[MessagePackObject] public class MsFormattedDerived : MsFormattedBase { [Key(1)] public int B { get; set; } }
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue