Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
acad89c713 | ||
|
|
bcc3052aa0 | ||
|
|
e584551fa3 | ||
|
|
3adb7c242a | ||
|
|
f0a81bb6fe | ||
|
|
8e218b04e8 |
73 changed files with 428 additions and 2790 deletions
6
.github/workflows/cultnet-interop.yml
vendored
6
.github/workflows/cultnet-interop.yml
vendored
|
|
@ -29,12 +29,6 @@ jobs:
|
|||
steps:
|
||||
- name: Check out CultLib
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# packages/cultnet-py/tests/test_declared_dependency_ranges.py reads the cultcache-py
|
||||
# release tags and their file contents with `git show`; a shallow checkout has no tags
|
||||
# and the test skips itself.
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Set up .NET
|
||||
uses: actions/setup-dotnet@v4
|
||||
|
|
|
|||
|
|
@ -1620,7 +1620,7 @@ The one door per runtime, and who walks through it:
|
|||
- **Shared paths.** Open, flush, commit, `ReadDurable`, `ReplaceDurable`. Rust `pull_all` RS:1243 also checks `is_absent` on the store and on the lock path before it takes the shared lock; that skips lock creation only, and the verdict stays with the reader.
|
||||
- **Cut line.** `merge-r1` h2 (C#) and h4/h6 (TS `readStoreBytes`); `mesh-single-file-reader` h1.
|
||||
- **Pinned by.** C#: `StoreReadabilityTests` `ADanglingFileLinkIsAnIoErrorOnOpenNotAnEmptyStore` :208, `ADanglingDirectoryLinkIsAnIoErrorOnOpenNotAnEmptyStore` :185, and the commit variants :196, :219. Mesh: `ADanglingLinkIsAnIoErrorOnTypedWriteRawWriteAndReadNeverAMissingFile` :107 and `OnlyNothingAtThePathIsAMissingFile` :134 (`CultMeshSingleFileStoreTests.cs`). Rust: `a_dangling_symlink_at_the_store_path_is_an_error_and_nothing_is_written` RS:5832. TS: "readStoreBytes reports nothing at the path only when the lstat finds nothing too".
|
||||
- **Known gaps.** The writer no longer tolerates a dangling link: the reader's refusal stands for every write, whole-store `pushAll` included (Rust `a_push_all_through_a_dangling_link_is_refused_like_push_and_writes_nothing`, TS `a pushAll through a dangling link is refused like push`; `variants:follow_up:writer-dangling-link-tolerance`). What a replace does to a live link is `variants:follow_up:store-replace-cut` item 2. Windows junction behavior was proven at `42f05f66` and `b1ff2dcb`, not at the final head.
|
||||
- **Known gaps.** The writer side still tolerates a dangling link: Rust `write_all_unlocked` RS:992-996 and TS `pushAll` TS:94-100 replace it on a whole-store write (`variants:follow_up:writer-dangling-link-tolerance`, owner `c2a-write-set`). What a replace does to a live link is `variants:follow_up:store-replace-cut` item 2. Windows junction behavior was proven at `42f05f66` and `b1ff2dcb`, not at the final head.
|
||||
|
||||
### C. Zero-byte file
|
||||
|
||||
|
|
@ -1672,7 +1672,7 @@ The one door per runtime, and who walks through it:
|
|||
- **Shared paths.** TS `pushAll` and Rust `push_all` only.
|
||||
- **Cut line.** `merge-r1` h4 (TS) and h6 (Rust).
|
||||
- **Pinned by.** The manifest walks run a soft flush over every vector in TS (the `softPushAll` operation, TS test :2336) and Rust (`soft_push_all`, RS:5149), each with an absent and a dangling-link case; `a_soft_push_all_writes_where_nothing_is_and_refuses_a_dangling_link` RS:6019.
|
||||
- **Known gaps.** C# and Python have no soft decision, and the C# Mesh flag is dead; no follow-up is registered (h6 `undone` names it). `writer-dangling-link-tolerance` is settled: the non-soft branch refuses a dangling link too.
|
||||
- **Known gaps.** C# and Python have no soft decision, and the C# Mesh flag is dead; no follow-up is registered (h6 `undone` names it). `writer-dangling-link-tolerance` applies to the non-soft branch.
|
||||
|
||||
### What no cut owns yet
|
||||
|
||||
|
|
|
|||
|
|
@ -107,8 +107,8 @@ Authority map (also in the spec):
|
|||
- `variants:follow_up:public-store-writes-behind-cache`: **leave**. It asks who may stage behind an attached cache (API
|
||||
shape); this cut changes what any staging is judged by. No collision: direct `Push`/`CommitBatch` reach the same on-disk
|
||||
judge, so they get the per-key rule for free. Hands must not give direct staging a different predicate.
|
||||
- `variants:follow_up:writer-dangling-link-tolerance`: **settled** (the reader owns the verdict; every runtime's whole-store
|
||||
`push_all` refuses a dangling link, as `push` does).
|
||||
- `variants:follow_up:writer-dangling-link-tolerance`: **leave**. Rust/TS whole-store `push_all` reading a dangling link is
|
||||
a reader/writer tolerance question, not ownership; this cut touches no leaf.
|
||||
|
||||
## Deferred doc findings this cut fixes
|
||||
|
||||
|
|
|
|||
6
package-lock.json
generated
6
package-lock.json
generated
|
|
@ -1205,7 +1205,7 @@
|
|||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@gamecult/cultcache-ts": "^0.15.0",
|
||||
"@gamecult/cultnet-ts": "^0.4.0"
|
||||
"@gamecult/cultnet-ts": "^0.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gamecult/cultmesh-ts": "^0.2.0",
|
||||
|
|
@ -1222,7 +1222,7 @@
|
|||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@gamecult/cultcache-ts": "^0.15.0",
|
||||
"@gamecult/cultnet-ts": "^0.4.0",
|
||||
"@gamecult/cultnet-ts": "^0.3.0",
|
||||
"koffi": "3.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -1236,7 +1236,7 @@
|
|||
},
|
||||
"packages/cultnet-ts": {
|
||||
"name": "@gamecult/cultnet-ts",
|
||||
"version": "0.4.0",
|
||||
"version": "0.3.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@gamecult/cultcache-ts": "^0.15.0",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
"scripts": {
|
||||
"build:ts": "node scripts/run-typescript-workspaces.mjs build",
|
||||
"test:ts": "node scripts/run-typescript-workspaces.mjs test",
|
||||
"test:pack:ts": "node --test scripts/test-declared-dependency-ranges.test.mjs && node scripts/test-typescript-package-closure.mjs && node scripts/test-declared-dependency-ranges.mjs",
|
||||
"test:pack:ts": "node scripts/test-typescript-package-closure.mjs && node scripts/test-declared-dependency-ranges.mjs",
|
||||
"test:eve-browser-network": "node scripts/verify-eve-browser-network.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,6 @@
|
|||
|
||||
## Unreleased
|
||||
|
||||
### Added
|
||||
|
||||
- `read_payload_tree(type_name, payload, decoder=None)`: the exported owner of the one-value, depth-first payload read that `decode_payload` uses, for payloads read outside a definition (`cultnet-py`'s witness bundle decode needs it, so `cultnet-py` now requires `cultcache-py>=0.4.0`). The manifest version is 0.4.0, the minor these Unreleased breaking changes imply, set in this change so the workspace install resolves the new bound; the `## Unreleased` heading becomes `## [0.4.0]` when it is tagged.
|
||||
|
||||
### Changed
|
||||
|
||||
- `msgpack` is a hard dependency, as `pyproject.toml` declared it. The optional-import fallbacks and the `cultcache-py[msgpack]` extra they named (which never existed) are gone: a missing msgpack is an `ImportError` at import.
|
||||
|
|
@ -14,7 +10,7 @@
|
|||
### Breaking
|
||||
|
||||
- A record whose payload carries a slot past its reader's declared slots, at any depth (the document, a nested record, a list element, a union arm, a unit arm), is read, held read-only, and every write or removal of its key (`put`, `put_global`, `update`, `put_envelope`, `put_envelopes`, `delete`, `delete_global`) is refused with `SchemaConflictError` before any store is touched (`cultlib-gaps:ruling:extra-trailing-slots`). Reading dropped those slots silently and the next write shed them. `SchemaConflictError.unread_slots` is an `UnreadSlots(type_name, slot_count, declared_count, occurrences, depth0_only)` and the message names the type and both counts, never a slot value. A depth-0 mark is consented, and the record writable, only when the record is stored under an id the document lists in `compatible_schema_ids` and that id's catalog entry has slots past the document's, covering the mark's slot count (`cultlib-gaps:ruling:slot-mark-consent`); a nested mark, or a mark under the document's own id, keeps the record read-only whatever is declared. Missing trailing slots keep their declared defaults. An untouched marked record keeps its stored bytes through every other write. Added `CultCache.is_writable(document, key)` (advisory; the write still decides: it also answers False for a foreign-carried key, for a key held as a type the document does not own, and for a global document at a non-global key; `delete` asks the same judges, so removing a key held as another type is refused with `SchemaConflictError` unless the document owns that record's schema id (its own or one it lists compatible), in which case the removal removes that record, its file row and its lookups, as a write of the key would replace it; it returned silently and removed nothing), `DocumentDefinition.decode_payload_marked`, and `UnreadSlots`. A document with a custom `decode` (no slot codec) never marks. A store holding a variant record is still unreadable here. Implies a minor bump while the version is 0.x (0.3.0 to 0.4.0): code that wrote back a record from a newer writer now gets `SchemaConflictError`.
|
||||
- A payload is one MessagePack value: `DocumentDefinition.decode_payload` refuses bytes after it, by count, with `ValueError` `<type> payload has N bytes after its first MessagePack value (refused on read).` (never the bytes, never the depth refusal; a payload that ends early stays the decoder's), as the C# reference does (`tests/vectors/union-parity/bytes-after-outcomes.msgpack`). It raised `msgpack.ExtraData` (a `ValueError`, so only a caller naming `msgpack.ExtraData` is affected); this is a minor-version bump while pre-1.0. The refused set is unchanged. Cache loads, `get` and every definition built on the MessagePack decoder go through it; a user `payload_decoder` that raises `ExtraData` is refused the same way. Depth is judged first, so a value nested past 256 with a tail is the depth refusal, as in C#.
|
||||
|
||||
- A cache holds one record per key whatever its type (`variants:ruling:cache-refuses-retype`). A `put`, `put_envelope`, `put_envelopes` or `put_global` of a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store, and changes nothing; retype a key by deleting it, then putting the new type. A write may replace another registered type's record only when the written type owns its schema id (declares it or lists it compatible) and both types share one home store. A load that reads one key from two stores, whatever the types, is refused with `CultCacheError` naming the key and both types, and admits nothing. A store that holds a record no registered type claims now loads and is carried: the cache does not serve it, the store keeps its bytes through every write, `delete` or `put` at its key is refused with `SchemaConflictError`, and registering a type that declares its schema id claims it at the next load. It raised `CultCacheError` "Unknown persisted document type". Two global types share the key `__global__`, so a second global type in one cache is now refused, and a store that already holds two global types at `__global__` (a JSONL store written by caches that never pulled could produce one) no longer loads: the pull raises `CultCacheError` and admits nothing. C# refuses a second persisted global on pull the same way (`cultcache-store-composition.md`, "Globals never invented"); repair such a file by deleting one of the two records with the runtime that wrote it.
|
||||
- Added `foreign_records()` and the frozen `ForeignRecord(key, schema_id, schema_name, stored_at)`, sorted by key in UTF-16 code-unit order. `stores.retyped` and `stores.written_owns` lose their underscore: the cache judge and the single-file store's commit guard ask one predicate.
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ discovery, sessions, local server) build on it, mirroring `cultcache-rs` /
|
|||
`[format, schemaCatalog, records]`
|
||||
- records store raw MessagePack payload bytes under schema-catalog identity
|
||||
- payloads decode only through the registered document definition for that type
|
||||
- a payload is exactly one MessagePack value: bytes after it are refused on read, by count
|
||||
- a cache holds one record per key whatever its type: a write of a second type at a held key is refused with
|
||||
`SchemaConflictError` (in memory too), and two stores holding one key refuse the load
|
||||
- a store record no registered type claims loads and is carried as foreign: `cache.foreign_records()` lists it, the store
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||
|
||||
[project]
|
||||
name = "cultcache-py"
|
||||
version = "0.4.0"
|
||||
version = "0.3.0"
|
||||
description = "Python CultCache runtime for GameCult typed state."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from .documents import (
|
|||
define_document_type,
|
||||
define_record_codec,
|
||||
define_union,
|
||||
read_payload_tree,
|
||||
)
|
||||
from .leaves import SlotCodec
|
||||
from .stores import JsonLinesBackingStore, SchemaConflictError, SingleFileMessagePackBackingStore, StoreUnreadableError
|
||||
|
|
@ -41,7 +40,6 @@ __all__ = [
|
|||
"StoreUnreadableError",
|
||||
"UnionArm",
|
||||
"UnreadSlots",
|
||||
"read_payload_tree",
|
||||
"leaves",
|
||||
"database_entry_field",
|
||||
"define_database_entry_type",
|
||||
|
|
|
|||
|
|
@ -158,26 +158,6 @@ def _messagepack_payload_decoder(payload: bytes) -> Any:
|
|||
return msgpack.unpackb(payload, raw=False)
|
||||
|
||||
|
||||
def read_payload_tree(type_name: str, payload: bytes, decoder: Callable[[bytes], Any] | None = None) -> Any:
|
||||
"""The one read of a payload's tree: exactly one MessagePack value, within the nesting limit. Depth is judged first,
|
||||
on the value read so far, so a tail never pre-empts it (C# throws at container 257 before it reaches the tail); then a
|
||||
tail is refused by count and never echoed. Raised outside the handler so no `__context__` keeps the tail."""
|
||||
after = -1
|
||||
unpacked: Any = None
|
||||
try:
|
||||
tree = (decoder or _messagepack_payload_decoder)(payload)
|
||||
except _STACK_FAULTS:
|
||||
raise CultCachePayloadDepthError(type_name, "read") from None
|
||||
except msgpack.ExtraData as error:
|
||||
after = len(error.extra)
|
||||
unpacked = error.unpacked
|
||||
if after >= 0:
|
||||
_refuse_past_depth_limit(unpacked, type_name, "read")
|
||||
raise ValueError(f"{type_name} payload has {after} bytes after its first MessagePack value (refused on read).")
|
||||
_refuse_past_depth_limit(tree, type_name, "read")
|
||||
return tree
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatabaseEntryField:
|
||||
name: str
|
||||
|
|
@ -254,7 +234,11 @@ class DocumentDefinition(Generic[T]):
|
|||
def decode_payload_marked(self, payload: bytes) -> tuple[T, UnreadSlots | None]:
|
||||
"""The value and what the decode found past the declared slots, at any depth. The payload depth check runs first: a
|
||||
payload past the limit is refused, never marked."""
|
||||
tree = read_payload_tree(self.type, payload, self.payload_decoder)
|
||||
try:
|
||||
tree = self.payload_decoder(payload)
|
||||
except _STACK_FAULTS:
|
||||
raise CultCachePayloadDepthError(self.type, "read") from None
|
||||
_refuse_past_depth_limit(tree, self.type, "read")
|
||||
outer, opened = _current_scope(), _SlotScope()
|
||||
_scope.open = opened
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ class JsonLinesBackingStore:
|
|||
"stored_at": envelope.stored_at,
|
||||
}, ensure_ascii=False, sort_keys=True))
|
||||
temp.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
|
||||
_replace_store(temp, self.path)
|
||||
temp.replace(self.path)
|
||||
|
||||
|
||||
class SingleFileMessagePackBackingStore:
|
||||
|
|
@ -323,7 +323,7 @@ def commit_store_file(path: Path, data: bytes) -> None:
|
|||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
_replace_store(staged, path)
|
||||
os.replace(staged, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(staged)
|
||||
|
|
@ -338,108 +338,6 @@ def commit_store_file(path: Path, data: bytes) -> None:
|
|||
os.close(directory)
|
||||
|
||||
|
||||
REPLACE_ATTEMPTS = 25
|
||||
REPLACE_RETRY_SECONDS = 0.02
|
||||
_ERROR_ACCESS_DENIED = 5
|
||||
_ERROR_SHARING_VIOLATION = 32
|
||||
_ERROR_LOCK_VIOLATION = 33
|
||||
_POSIX_UNSUPPORTED = (1, 50, 87) # incorrect function, not supported, invalid parameter: FAT, exFAT, before Windows 10 1607
|
||||
MOVEFILE_REPLACE_WRITE_THROUGH = 0x9 # MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH
|
||||
|
||||
|
||||
class ReplaceKernel:
|
||||
"""The kernel32 calls a Windows store replace needs, each answering 0 or a Win32 error code."""
|
||||
|
||||
def rename_posix(self, staged: str, destination: str) -> int:
|
||||
"""CreateFileW(staged, DELETE|SYNCHRONIZE), then FileRenameInfoEx with REPLACE_IF_EXISTS|POSIX_SEMANTICS, then CloseHandle."""
|
||||
raise NotImplementedError
|
||||
|
||||
def move_file_ex(self, staged: str, destination: str, flags: int) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
def sleep(self, seconds: float) -> None:
|
||||
time.sleep(seconds)
|
||||
|
||||
|
||||
def _replace_store(staged: str | os.PathLike[str], destination: str | os.PathLike[str]) -> None:
|
||||
"""Replaces `destination` with `staged`: POSIX-semantics rename first on Windows (so a reader holding the store open
|
||||
never sees it absent), os.replace elsewhere."""
|
||||
if sys.platform == "win32":
|
||||
replace_store_windows(os.fspath(staged), os.fspath(destination), WindowsReplaceKernel())
|
||||
else:
|
||||
os.replace(staged, destination)
|
||||
|
||||
|
||||
def replace_store_windows(staged: str, destination: str, kernel: ReplaceKernel) -> None:
|
||||
"""POSIX-semantics rename, falling back for the rest of the call to write-through MoveFileExW where POSIX semantics are
|
||||
unsupported; errors 5, 32 and 33 retry up to 25 times 20 ms apart, any other code raises at once. The TypeScript and C#
|
||||
runtimes take the same decision table."""
|
||||
posix = True
|
||||
last = 0
|
||||
for _ in range(REPLACE_ATTEMPTS):
|
||||
code = kernel.rename_posix(staged, destination) if posix else kernel.move_file_ex(staged, destination, MOVEFILE_REPLACE_WRITE_THROUGH)
|
||||
if posix and code in _POSIX_UNSUPPORTED:
|
||||
posix = False
|
||||
code = kernel.move_file_ex(staged, destination, MOVEFILE_REPLACE_WRITE_THROUGH)
|
||||
if code == 0:
|
||||
return
|
||||
if code not in (_ERROR_ACCESS_DENIED, _ERROR_SHARING_VIOLATION, _ERROR_LOCK_VIOLATION):
|
||||
raise _replace_error(f"CultCache could not replace the store: Win32 error {code}.", code)
|
||||
last = code
|
||||
kernel.sleep(REPLACE_RETRY_SECONDS)
|
||||
raise _replace_error(f"CultCache could not replace the store after {REPLACE_ATTEMPTS} attempts: last Win32 error {last}.", last)
|
||||
|
||||
|
||||
def _replace_error(message: str, code: int) -> OSError:
|
||||
error = OSError(message)
|
||||
error.winerror = code # type: ignore[attr-defined]
|
||||
return error
|
||||
|
||||
|
||||
class WindowsReplaceKernel(ReplaceKernel):
|
||||
"""kernel32 bound by ctypes, as _try_lock_windows binds it. Windows only."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
self._ctypes = ctypes
|
||||
k = self._k = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
k.CreateFileW.restype = wintypes.HANDLE
|
||||
k.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p, wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE]
|
||||
k.SetFileInformationByHandle.restype = wintypes.BOOL
|
||||
k.SetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
|
||||
k.CloseHandle.restype = wintypes.BOOL
|
||||
k.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
k.MoveFileExW.restype = wintypes.BOOL
|
||||
k.MoveFileExW.argtypes = [wintypes.LPCWSTR, wintypes.LPCWSTR, wintypes.DWORD]
|
||||
|
||||
def rename_posix(self, staged: str, destination: str) -> int:
|
||||
ctypes, k = self._ctypes, self._k
|
||||
delete_synchronize, share_all, open_existing, normal = 0x00110000, 0x7, 3, 0x80
|
||||
handle = k.CreateFileW(staged, delete_synchronize, share_all, None, open_existing, normal, None)
|
||||
if handle is None or ctypes.c_void_p(handle).value in (None, ctypes.c_void_p(-1).value):
|
||||
return ctypes.get_last_error()
|
||||
try:
|
||||
# FILE_RENAME_INFO: Flags (DWORD), RootDirectory (pointer-aligned HANDLE), FileNameLength (DWORD), FileName (WCHAR[]).
|
||||
name = os.path.abspath(destination).encode("utf-16-le")
|
||||
pointer = ctypes.sizeof(ctypes.c_void_p)
|
||||
length_at = pointer + pointer
|
||||
info = ctypes.create_string_buffer(length_at + 4 + len(name) + 2)
|
||||
info[0:4] = (0x3).to_bytes(4, "little") # FILE_RENAME_FLAG_REPLACE_IF_EXISTS | FILE_RENAME_FLAG_POSIX_SEMANTICS
|
||||
info[length_at:length_at + 4] = len(name).to_bytes(4, "little")
|
||||
info[length_at + 4:length_at + 4 + len(name)] = name
|
||||
file_rename_info_ex = 22
|
||||
if k.SetFileInformationByHandle(handle, file_rename_info_ex, info, len(info)):
|
||||
return 0
|
||||
return ctypes.get_last_error()
|
||||
finally:
|
||||
k.CloseHandle(handle)
|
||||
|
||||
def move_file_ex(self, staged: str, destination: str, flags: int) -> int:
|
||||
return 0 if self._k.MoveFileExW(staged, destination, flags) else self._ctypes.get_last_error()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def store_write_lock(path: Path, timeout: float = LOCK_TIMEOUT_SECONDS) -> Iterator[None]:
|
||||
"""Holds the exclusive store lock on `<path>.lock`, the lock the C#, Rust and TypeScript runtimes take: flock on Unix,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from cultcache_py import (
|
|||
SlotCodec,
|
||||
SchemaConflictError,
|
||||
SingleFileMessagePackBackingStore,
|
||||
StoreUnreadableError,
|
||||
UnionArm,
|
||||
UnreadSlots,
|
||||
database_entry_field,
|
||||
|
|
@ -179,22 +178,6 @@ class StoreRowTests(unittest.TestCase):
|
|||
return slot_unit_document
|
||||
return declaring_document if store == "extra-slot-consent.cc" else surface_document
|
||||
|
||||
def test_the_variant_row_is_refused_as_a_store_not_walked_and_the_rest_of_the_store_reads_without_it(self) -> None:
|
||||
# This runtime holds no variant records (ruling: refuse, not carry; the reference's marked-variant row, the one
|
||||
# row with a flattenRefusedBy, is the C# reader's). A five-slot persisted record makes the whole store
|
||||
# unreadable, the file is left as it was, and the store without exactly that row reads.
|
||||
variants = [row for row in _rows() if row[8] is not None]
|
||||
self.assertEqual([(row[0], row[1], row[8]) for row in variants], [("extra-slot-store.cc", "marked-variant", "marked")])
|
||||
path = self.tmp / "extra-slot-store.cc"
|
||||
shutil.copyfile(VECTORS / "extra-slot-store.cc", path)
|
||||
before = path.read_bytes()
|
||||
with self.assertRaises(StoreUnreadableError):
|
||||
_open(path, surface_document, slot_unit_document)
|
||||
self.assertEqual(path.read_bytes(), before)
|
||||
_, cache = self._store("extra-slot-store.cc")
|
||||
self.assertIsNone(cache.get(surface_document, "marked-variant"))
|
||||
self.assertIsNotNone(cache.get(surface_document, "marked"))
|
||||
|
||||
def test_every_row_is_writable_exactly_where_the_vector_says_and_refuses_with_its_mark(self) -> None:
|
||||
walked = 0
|
||||
for store, key, writable, explained, type_name, carried, declared, occurrences, flatten in _rows():
|
||||
|
|
|
|||
|
|
@ -1,124 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from cultcache_py import SingleFileMessagePackBackingStore, stores
|
||||
from cultcache_py.backing_store import CultCacheEnvelope
|
||||
from cultcache_py.stores import MOVEFILE_REPLACE_WRITE_THROUGH, ReplaceKernel, replace_store_windows
|
||||
|
||||
|
||||
class FakeKernel(ReplaceKernel):
|
||||
def __init__(self, posix: list[int], move_ex: list[int]) -> None:
|
||||
self.posix, self.move_ex = list(posix), list(move_ex)
|
||||
self.log: list[str] = []
|
||||
self.sleeps: list[float] = []
|
||||
|
||||
def rename_posix(self, staged, destination):
|
||||
self.log.append("posix")
|
||||
return self.posix.pop(0) if self.posix else 0
|
||||
|
||||
def move_file_ex(self, staged, destination, flags):
|
||||
self.log.append(f"moveex:{flags}")
|
||||
return self.move_ex.pop(0) if self.move_ex else 0
|
||||
|
||||
def sleep(self, seconds):
|
||||
self.sleeps.append(seconds)
|
||||
|
||||
|
||||
def run(posix, move_ex):
|
||||
kernel = FakeKernel(posix, move_ex)
|
||||
return kernel, lambda: replace_store_windows("a", "b", kernel)
|
||||
|
||||
|
||||
class WindowsReplaceTable(unittest.TestCase):
|
||||
def test_falls_back_to_write_through_move_where_posix_rename_is_unsupported(self):
|
||||
self.assertEqual(MOVEFILE_REPLACE_WRITE_THROUGH, 0x9)
|
||||
for unsupported in (1, 50, 87):
|
||||
kernel, go = run([unsupported], [0])
|
||||
go()
|
||||
self.assertEqual(kernel.log, ["posix", "moveex:9"])
|
||||
|
||||
def test_once_on_move_the_rest_of_the_call_stays_there(self):
|
||||
kernel, go = run([87], [32, 0])
|
||||
go()
|
||||
self.assertEqual(kernel.log, ["posix", "moveex:9", "moveex:9"])
|
||||
|
||||
def test_transient_codes_retry_the_same_primitive(self):
|
||||
for transient in (5, 32, 33):
|
||||
kernel, go = run([transient, transient, 0], [])
|
||||
go()
|
||||
self.assertEqual(kernel.log, ["posix", "posix", "posix"])
|
||||
self.assertEqual(kernel.sleeps, [0.02, 0.02])
|
||||
|
||||
def test_a_working_posix_rename_is_the_whole_replace(self):
|
||||
kernel, go = run([0], [])
|
||||
go()
|
||||
self.assertEqual(kernel.log, ["posix"])
|
||||
self.assertEqual(kernel.sleeps, [])
|
||||
|
||||
def test_unsupported_after_the_fallback_is_an_error_not_another_fallback(self):
|
||||
kernel, go = run([87], [32, 87])
|
||||
with self.assertRaisesRegex(OSError, "Win32 error 87") as caught:
|
||||
go()
|
||||
self.assertEqual(caught.exception.winerror, 87)
|
||||
self.assertEqual(kernel.log, ["posix", "moveex:9", "moveex:9"])
|
||||
for unsupported in (1, 50, 87):
|
||||
kernel, go = run([87], [unsupported])
|
||||
with self.assertRaisesRegex(OSError, f"Win32 error {unsupported}"):
|
||||
go()
|
||||
self.assertEqual(kernel.log, ["posix", "moveex:9"])
|
||||
|
||||
def test_gives_up_after_25_attempts_naming_the_last_code(self):
|
||||
kernel, go = run([32] * 30, [])
|
||||
with self.assertRaisesRegex(OSError, "after 25 attempts: last Win32 error 32\\."):
|
||||
go()
|
||||
self.assertEqual(len(kernel.log), 25)
|
||||
|
||||
def test_other_codes_fail_at_once(self):
|
||||
kernel, go = run([2], [])
|
||||
with self.assertRaisesRegex(OSError, "Win32 error 2"):
|
||||
go()
|
||||
self.assertEqual(len(kernel.log), 1)
|
||||
kernel, go = run([87], [3])
|
||||
with self.assertRaisesRegex(OSError, "Win32 error 3"):
|
||||
go()
|
||||
|
||||
|
||||
class StoreWritesRouteThroughTheReplace(unittest.TestCase):
|
||||
def test_both_stores_replace_through_replace_store(self):
|
||||
seen: list[tuple[Path, Path]] = []
|
||||
real = stores._replace_store
|
||||
|
||||
def spy(staged, destination):
|
||||
seen.append((Path(staged), Path(destination)))
|
||||
real(staged, destination)
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory, mock.patch.object(stores, "_replace_store", spy):
|
||||
envelope = CultCacheEnvelope(key="k", type="t", payload=b"\x01", stored_at="2026-11-06T00:00:00Z")
|
||||
binary = Path(directory) / "s.cc"
|
||||
SingleFileMessagePackBackingStore(binary).push_all([envelope])
|
||||
lines = Path(directory) / "s.jsonl"
|
||||
stores.JsonLinesBackingStore(lines).push_all([envelope])
|
||||
self.assertEqual([destination for _, destination in seen], [binary, lines])
|
||||
self.assertTrue(binary.read_bytes() and lines.read_text())
|
||||
|
||||
@unittest.skipUnless(sys.platform == "win32", "needs a real Windows")
|
||||
def test_real_kernel_replaces_through_both_primitives(self):
|
||||
kernel = stores.WindowsReplaceKernel()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
for name, replace in (("posix", kernel.rename_posix), ("moveex", lambda s, d: kernel.move_file_ex(s, d, MOVEFILE_REPLACE_WRITE_THROUGH))):
|
||||
staged, store = Path(directory, name + ".tmp"), Path(directory, name)
|
||||
store.write_bytes(b"old")
|
||||
staged.write_bytes(b"new")
|
||||
self.assertEqual(replace(str(staged), str(store)), 0)
|
||||
self.assertEqual(store.read_bytes(), b"new")
|
||||
self.assertEqual(kernel.rename_posix(str(Path(directory, "missing")), str(Path(directory, "x"))), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -661,78 +661,6 @@ def _as_tuple(tree: object) -> object:
|
|||
return tuple(_as_tuple(child) for child in tree) if isinstance(tree, list) else tree
|
||||
|
||||
|
||||
|
||||
class BytesAfterVectorTests(unittest.TestCase):
|
||||
CLAUSE = "bytes after its first MessagePack value (refused on read)"
|
||||
|
||||
def _assert_chain_clean(self, error: BaseException, file: str) -> None:
|
||||
link: BaseException | None = error
|
||||
while link is not None:
|
||||
self.assertNotIn(CANARY, str(link), file)
|
||||
self.assertNotIsInstance(link, msgpack.ExtraData, file)
|
||||
link = link.__cause__ or link.__context__
|
||||
|
||||
def test_every_bytes_after_vector_is_refused(self) -> None:
|
||||
base = _vector("union-surface.msgpack")
|
||||
entries = msgpack.unpackb(_vector("bytes-after-outcomes.msgpack"), raw=False)
|
||||
self.assertEqual(len(entries), 4)
|
||||
for file, outcome_kind, union, key, detail in entries:
|
||||
self.assertEqual((outcome_kind, union, key, detail), ("refused", "UnionSurface", None, "bytes-after"), file)
|
||||
payload = _vector(file)
|
||||
self.assertTrue(payload.startswith(base), file)
|
||||
count = len(payload) - len(base)
|
||||
with self.assertRaises(ValueError, msg=file) as caught:
|
||||
surface_document.decode_payload(payload)
|
||||
error = caught.exception
|
||||
self.assertIs(type(error), ValueError, file)
|
||||
self.assertNotIsInstance(error, CultCachePayloadDepthError, file)
|
||||
self.assertEqual(
|
||||
str(error), f"{surface_document.type} payload has {count} {self.CLAUSE}.", file
|
||||
)
|
||||
self.assertNotIn("the limit is", str(error), file)
|
||||
self._assert_chain_clean(error, file)
|
||||
|
||||
def test_a_single_file_store_holding_a_tail_as_a_record_payload_refuses_on_load(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "tail.cc"
|
||||
shutil.copyfile(VECTORS / "union-surface.cc", path)
|
||||
store = SingleFileMessagePackBackingStore(path)
|
||||
[envelope] = store.pull_all()
|
||||
store.push(dataclasses.replace(envelope, payload=_vector("bytes-after-nil.msgpack")))
|
||||
cache = (
|
||||
CultCache.builder()
|
||||
.register_document_type(surface_document)
|
||||
.add_generic_store(SingleFileMessagePackBackingStore(path))
|
||||
.build()
|
||||
)
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
cache.pull_all_backing_stores()
|
||||
self.assertIn(f"1 {self.CLAUSE}", str(caught.exception))
|
||||
self.assertIn(surface_document.type, str(caught.exception))
|
||||
self.assertIsNone(cache.get(surface_document, "surface"))
|
||||
|
||||
def test_depth_wins_over_a_tail_whatever_the_unpackers_own_stack_limit(self) -> None:
|
||||
# Vector-independent: a value nested past 256 is the depth refusal even with a tail, as C# throws at container 257
|
||||
# before it reaches the tail; 2000 is past msgpack's own stack, so the winner is not a library constant.
|
||||
for depth in (257, 300, 900, 2000, 100000):
|
||||
for tail in (b"\xc0", b"\xc1\xc1"):
|
||||
with self.subTest(depth=depth, tail=len(tail)):
|
||||
with self.assertRaises(CultCachePayloadDepthError) as caught:
|
||||
surface_document.decode_payload(b"\x91" * depth + b"\xc0" + tail)
|
||||
self.assertIn("refused on read", str(caught.exception))
|
||||
for depth in (1, 255, 256):
|
||||
with self.subTest(at=depth):
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
surface_document.decode_payload(b"\x91" * depth + b"\xc0\xc0")
|
||||
self.assertNotIsInstance(caught.exception, CultCachePayloadDepthError)
|
||||
self.assertIn(f"1 {self.CLAUSE}", str(caught.exception))
|
||||
|
||||
def test_a_payload_that_ends_early_is_the_decoders_not_bytes_after(self) -> None:
|
||||
truncated = _vector("union-surface.msgpack")[:-1]
|
||||
with self.assertRaises(Exception) as caught:
|
||||
surface_document.decode_payload(truncated)
|
||||
self.assertNotIn("bytes after", str(caught.exception))
|
||||
|
||||
class PayloadDepthRuleTests(unittest.TestCase):
|
||||
"""The nesting rule at its seam, on definitions that are not unions: the same routine serves any payload codec."""
|
||||
|
||||
|
|
|
|||
|
|
@ -11,13 +11,11 @@ only in git history.
|
|||
Semver: this is a breaking change in the 0.x line (0.3.x to 0.4.0).
|
||||
|
||||
- A record whose payload carries unread slots is read-only. A payload with a slot past its type's declared slots, at any depth (record, nested record, list element, union arm, unit arm), still decodes, and every write or removal of its key is now refused before any store is touched: `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch` (the whole batch) and `delete`, so `update` too. The refusal is `SchemaConflictError` with the new `unread_slots`; its message names the type and both slot counts and never a slot value. Holders of such a record used to overwrite it and drop the extra slots silently. The payload nesting refusal still runs first. A held payload this build cannot read at all (an arm it lacks from a newer writer, or a corrupt record) is read-only too: every write or removal is refused with an error naming the type and key (a newer writer's record is read with that build; a corrupt one is restored from a backup or removed at the store itself, as no cache call removes it), and `is_writable` answers false. `load_soa` is a write of every held key it omits or changes, so it refuses to drop or replace a read-only record. `is_writable` also answers false for a key a store carries as a foreign record and for a type other than the one holding the key, as the writes do. Each write over a held key decodes the held payload once to ask (about 40% of a `put` of a 200,000-element record, measured in release); keys nothing holds pay nothing.
|
||||
- cultmesh-rs `pull_rudp_catalog_snapshot` applies a snapshot as one cache commit (`put_prepared_batch`), as C# does: a snapshot document for a key the node holds read-only, or any other refusal, applies nothing. It used to apply document by document, so a refusal ended the pull with the earlier documents kept. A snapshot whose types route to different backing stores is now refused whole, as a batch cannot span stores. C# `CultCache.Commit` refuses a cross-store commit the same way (`The batch spans A and B; a commit lands in one home store`), so the two agree; a test pins it.
|
||||
- cultmesh-rs `pull_rudp_catalog_snapshot` applies documents one by one with `put_raw_envelope`, so a snapshot document for a key the node holds read-only ends the pull at that document and the documents before it stay applied. C# applies a raw snapshot as one cache commit, so there a refusal applies nothing: this is a divergence, owed by follow-up `mesh-snapshot-atomic` (stage the pull through `put_prepared_batch`).
|
||||
- `SchemaConflictError` gains the public field `unread_slots: Option<UnreadSlots>`; code that builds it with a struct literal must set it (`None`).
|
||||
- A unit arm with slots (`[N, [extra...]]`) decodes as the unit arm and marks the record, where the derive refused it.
|
||||
- The registration map's value changes from the schema name to a private struct holding the schema name and the unread-slots check.
|
||||
- A cache whose last `pull_all_backing_stores` was refused (a tail, a payload past the nesting bound, a duplicate unique-index value, a retype conflict, a record from the wrong store) refuses every `put`, `put_envelope`, `put_raw_envelope`, `put_prepared_batch`, `update` and `delete` with `CultCacheLoadRefused` until a pull succeeds. Before, it stayed writable over an empty image, and the next batch write replaced the store with only the new records.
|
||||
- `CacheBackingStore::push_all` writes the given entries onto the store as it is when the write lands, under the store's lock or transaction: a record under the same identity is replaced and no other record is removed. It used to replace everything the store held, so a cache whose image was stale (another writer had added a record since its pull) wiped that record with `put_prepared_batch`. `put_prepared_batch` now hands the store its staged records alone, as C# `CommitBatch` and Python `push_all` do; the lay-back of foreign records is gone because they are never rewritten. A custom `CacheBackingStore` must give `push_all` merge semantics. `push_all` reads the store, so through a dangling link it is refused like `push` and no longer replaces the link; a soft flush is unchanged.
|
||||
- A cache with a backing store also refuses every write and removal, with `CultCacheLoadRefused`, until it has pulled: a cache that never pulled, or that gained a store since its last pull, holds no image of its stores, and a batch from it replaced a populated store with only the new records. `is_writable` answers false meanwhile. A cache with no store needs no pull. Callers that attach a store and write (cultmesh nodes created with `pull_on_start: false`) must call `pull_all_backing_stores` first.
|
||||
- `decode_payload`, `put_raw_envelope` and prepared batches refuse a payload with bytes after its first MessagePack value (`payload has {n} bytes after its first MessagePack value`), by count; the tail is never walked or quoted. They used to read the first value and keep or drop the tail, so a record with a stray tail was served and rewritten without it. A store holding such a record (single-file, redb, `load_soa`) is refused at load, naming the type, the key and the count, and stays unreadable until the record is repaired; the same check now runs on every record at load, as the C# and Python runtimes do at open. The nesting bound is part of that check, so a stored payload nested past it is refused at load too, where it used to load and refuse on read.
|
||||
- A single-file store written as `cultcache.store.v3` is read and written: a store the C# runtime marked v3 (it holds a record with an element id: an owned list element with a `[CultElementId]` member) keeps its header here, and is carried through every rewrite with its ids; this runtime never drops the mark. A runtime from before this change refuses a v3 store by name. Rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
||||
- A write over a file the reader refuses (a zero-byte file, a truncated store, a stamp this runtime does not read) refuses with `CultCacheStoreUnreadable` and leaves the file as it is. A whole-store write used to replace such a file. This reverses 0.3.1, where `push_all` replaced a file with no readable stamp (healing it); the stamp rule now has one owner, the reader every write asks under the lock, and it accepts v1 and v3 (0.3.1 refused v3).
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ CultCache is a domain cache with persistence adapters.
|
|||
file access with a sidecar lock file.
|
||||
|
||||
This is not an ORM, not a database, and not distributed consensus in a novelty
|
||||
hat. Several processes may write the same single-file store, one at a time under
|
||||
its sidecar lock; write ordering across keys or stores needs a coordinator.
|
||||
hat. If multiple processes write the same backing file, use an external lock or
|
||||
a coordinator.
|
||||
|
||||
## Current API
|
||||
|
||||
|
|
@ -384,9 +384,10 @@ an empty store. Any other failure to reach it, such as a symlink loop, a file
|
|||
where a parent directory should be, or a missing permission, is an I/O error.
|
||||
A store file that exists and holds zero bytes is `Undecodable`: no CultCache
|
||||
writer leaves one. A symbolic link whose target is gone is not a missing store:
|
||||
reading it is an I/O error, so a read-then-write (`push`, `push_all`, `delete`, the exchanges)
|
||||
fails before writing. Writes do not yet follow links: a write through a live link
|
||||
replaces the link with a regular file and leaves its target alone. A refusal names no
|
||||
reading it is an I/O error, so a read-then-write (`push`, `delete`, the exchanges)
|
||||
fails before writing. Writes do not yet follow links: a write through a live link,
|
||||
and a `push_all` through a dangling one, replace the link with a regular file and
|
||||
leave its target alone. A refusal names no
|
||||
value read from the file; it names a record only by its key and schema id, and
|
||||
a header only when it has the shape `cultcache.store.v<digits>`.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
use anyhow::Result;
|
||||
use anyhow::anyhow;
|
||||
use cultcache_rs::CultCache;
|
||||
use cultcache_rs::CultSlots;
|
||||
use cultcache_rs::CultUnion;
|
||||
use cultcache_rs::DatabaseEntry;
|
||||
use cultcache_rs::SingleFileMessagePackBackingStore;
|
||||
use serde_json::json;
|
||||
|
|
@ -41,100 +39,6 @@ struct CultCacheInteropDeck {
|
|||
marks: Vec<CultCacheInteropMark>,
|
||||
}
|
||||
|
||||
// A union field and nested bytes: the shapes a Bifrost-style store holds beyond plain strings.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, CultUnion)]
|
||||
enum InteropArm {
|
||||
#[cultcache(key = 0)]
|
||||
Text(InteropTextArm),
|
||||
#[cultcache(key = 1)]
|
||||
Pack(InteropPackArm),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, CultSlots)]
|
||||
struct InteropTextArm {
|
||||
#[cultcache(key = 0)]
|
||||
text: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, CultSlots)]
|
||||
struct InteropPackArm {
|
||||
#[cultcache(key = 0)]
|
||||
label: String,
|
||||
#[cultcache(key = 1)]
|
||||
chunks: Vec<serde_bytes::ByteBuf>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DatabaseEntry)]
|
||||
#[cultcache(type = "cultcache.interop-shape", schema = "CultCacheInteropShape")]
|
||||
struct CultCacheInteropShape {
|
||||
#[cultcache(key = 0)]
|
||||
document_id: String,
|
||||
#[cultcache(key = 1)]
|
||||
arm: InteropArm,
|
||||
#[cultcache(key = 2)]
|
||||
raw: serde_bytes::ByteBuf,
|
||||
}
|
||||
|
||||
fn shapes() -> Vec<CultCacheInteropShape> {
|
||||
vec![
|
||||
CultCacheInteropShape {
|
||||
document_id: "shape:text".into(),
|
||||
arm: InteropArm::Text(InteropTextArm { text: "union arm 0".into() }),
|
||||
raw: (0..=255u8).collect::<Vec<_>>().into(),
|
||||
},
|
||||
CultCacheInteropShape {
|
||||
document_id: "shape:pack".into(),
|
||||
arm: InteropArm::Pack(InteropPackArm {
|
||||
label: "union arm 1".into(),
|
||||
chunks: vec![vec![0xc0, 0xc1, 0x00].into(), vec![].into(), vec![0x92, 0x01, 0x02].into()],
|
||||
}),
|
||||
raw: vec![0xde, 0xad, 0xbe, 0xef].into(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn write_shapes(file: &str) -> Result<()> {
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<CultCacheInteropShape>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(file))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
for shape in shapes() {
|
||||
cache.put(&shape.document_id, &shape)?;
|
||||
}
|
||||
print_shapes(shapes())
|
||||
}
|
||||
|
||||
/// Opens the store through the read-only snapshot door: no lock file, no staging file, no rewrite.
|
||||
fn read_shapes(file: &str) -> Result<()> {
|
||||
let envelopes = SingleFileMessagePackBackingStore::new(file).pull_all_read_only_snapshot()?;
|
||||
let mut found = Vec::new();
|
||||
for envelope in envelopes.iter().filter(|envelope| envelope.r#type == CultCacheInteropShape::TYPE) {
|
||||
found.push(cultcache_rs::decode_payload::<CultCacheInteropShape>(&envelope.payload)?);
|
||||
}
|
||||
print_shapes(found)
|
||||
}
|
||||
|
||||
fn print_shapes(mut found: Vec<CultCacheInteropShape>) -> Result<()> {
|
||||
let hex = |bytes: &[u8]| bytes.iter().map(|byte| format!("{byte:02X}")).collect::<String>();
|
||||
found.sort_by(|a, b| a.document_id.cmp(&b.document_id));
|
||||
let rows: Vec<_> = found
|
||||
.iter()
|
||||
.map(|shape| {
|
||||
let arm = match &shape.arm {
|
||||
InteropArm::Text(text) => json!({ "kind": "text", "text": text.text }),
|
||||
InteropArm::Pack(pack) => json!({
|
||||
"kind": "pack",
|
||||
"label": pack.label,
|
||||
"chunks": pack.chunks.iter().map(|chunk| hex(chunk)).collect::<Vec<_>>(),
|
||||
}),
|
||||
};
|
||||
json!({ "documentId": shape.document_id, "arm": arm, "raw": hex(&shape.raw) })
|
||||
})
|
||||
.collect();
|
||||
println!("{}", serde_json::to_string(&rows)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_deck(file: &str) -> Result<()> {
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<CultCacheInteropDeck>()?;
|
||||
|
|
@ -166,8 +70,6 @@ fn main() -> Result<()> {
|
|||
"write" => write_note(file, require_arg(&options, "runtime-id")?)?,
|
||||
"read" => read_note(file)?,
|
||||
"read-deck" => read_deck(file)?,
|
||||
"write-shapes" => write_shapes(file)?,
|
||||
"read-shapes" => read_shapes(file)?,
|
||||
"hold-lock" => hold_lock(file, require_arg(&options, "ms")?.parse()?)?,
|
||||
_ => return Err(anyhow!("unknown mode {mode}")),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1253,14 +1253,12 @@ pub trait CacheBackingStore: Send {
|
|||
fn push(&mut self, entry: &CultCacheEnvelope) -> Result<()>;
|
||||
fn delete(&mut self, entry: &CultCacheEnvelope) -> Result<()>;
|
||||
|
||||
/// Writes `entries` onto the store as it is when the write lands, in one
|
||||
/// atomic step: a record under the same identity is replaced and no other
|
||||
/// record is removed or changed, so a writer whose image is stale cannot
|
||||
/// drop a record it never saw. On `Err` the store still holds exactly what
|
||||
/// it held before. A cache batch (`put_prepared_batch`) is all-or-nothing
|
||||
/// only through this. There is no default, because a write assembled from
|
||||
/// `push` calls leaves the store partial when one of them fails; a store
|
||||
/// without an atomic write does not compile:
|
||||
/// Replaces everything the store holds with `entries` in one atomic step:
|
||||
/// on `Err` the store still holds exactly what it held before. A cache
|
||||
/// batch (`put_prepared_batch`) is all-or-nothing only through this. There
|
||||
/// is no default, because a replace assembled from `delete` and `push`
|
||||
/// calls leaves the store emptied or partial when one of them fails; a
|
||||
/// store without an atomic replace does not compile:
|
||||
///
|
||||
/// ```compile_fail,E0046
|
||||
/// use cultcache_rs::{CacheBackingStore, CultCacheEnvelope};
|
||||
|
|
@ -1677,7 +1675,13 @@ impl SingleFileMessagePackBackingStore {
|
|||
// payloads this store passes through may hold ids), one that is gone or legacy is written v1. The file is read by
|
||||
// the same reader as an open, so a store this runtime cannot read completely, a variant included, is refused
|
||||
// and never overwritten.
|
||||
let disk = self.read_store_unlocked()?;
|
||||
let disk = match self.read_store_unlocked() {
|
||||
// A dangling link holds no store, so a whole-file replace has no header to keep and replaces the link (R3 pins
|
||||
// this). The only NotFound the reader returns is that one: a merging writer has already refused it when it
|
||||
// read the current entries.
|
||||
Err(error) if io_not_found(&error) => DiskStore::absent(),
|
||||
other => other?,
|
||||
};
|
||||
let entries = settle_one_record_per_key(entries, &disk)?;
|
||||
let bytes = encode_store_snapshot(&entries, &disk)
|
||||
.and_then(|snapshot| rmp_serde::to_vec(&snapshot).context("failed to encode MessagePack"))
|
||||
|
|
@ -1958,18 +1962,14 @@ impl CacheBackingStore for SingleFileMessagePackBackingStore {
|
|||
}
|
||||
|
||||
// Of entries sharing a key and a schema id the last one given is the record, whatever their type names (a type
|
||||
// renamed under a stable id); it replaces the held record under that key and schema id. A second schema id
|
||||
// under a key reaches the commit guard, which judges the merged store.
|
||||
// renamed under a stable id); a second schema id under a key reaches the commit guard.
|
||||
let mut by_id = BTreeMap::<(String, String), &CultCacheEnvelope>::new();
|
||||
for entry in entries {
|
||||
by_id.insert((schema_identity(entry).to_string(), entry.key.clone()), entry);
|
||||
}
|
||||
let mut merged = self.read_all_unlocked()?;
|
||||
let replaced: BTreeSet<_> = by_id.values().map(|entry| entry_id(entry)).collect();
|
||||
merged.retain(|held| !replaced.contains(&entry_id(held)));
|
||||
merged.extend(by_id.into_values().cloned());
|
||||
merged.sort_by_key(entry_id);
|
||||
self.write_all_unlocked(&merged)
|
||||
let mut entries: Vec<CultCacheEnvelope> = by_id.into_values().cloned().collect();
|
||||
entries.sort_by_key(entry_id);
|
||||
self.write_all_unlocked(&entries)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -2757,6 +2757,13 @@ impl CacheBackingStore for OwnedRedbMessagePackBackingStore {
|
|||
let write = self.inner.database.begin_write()?;
|
||||
{
|
||||
let mut table = write.open_table(REDB_ENVELOPES)?;
|
||||
let keys = table
|
||||
.iter()?
|
||||
.map(|row| Ok(row?.0.value().to_vec()))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
for key in keys {
|
||||
table.remove(key.as_slice())?;
|
||||
}
|
||||
for entry in entries {
|
||||
insert_redb_entry(&mut table, entry)?;
|
||||
}
|
||||
|
|
@ -2892,6 +2899,13 @@ impl CacheBackingStore for RedbMessagePackBackingStore {
|
|||
let write = database.begin_write()?;
|
||||
{
|
||||
let mut table = write.open_table(REDB_ENVELOPES)?;
|
||||
let keys = table
|
||||
.iter()?
|
||||
.map(|row| Ok(row?.0.value().to_vec()))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
for key in keys {
|
||||
table.remove(key.as_slice())?;
|
||||
}
|
||||
for entry in entries {
|
||||
insert_redb_entry(&mut table, entry)?;
|
||||
}
|
||||
|
|
@ -2923,24 +2937,13 @@ pub struct CultCache {
|
|||
/// Records a store holds that no registered type claims: (store index, the envelope as the store read it). Derived from the last
|
||||
/// pull and replaced with it; written nowhere else. It decides only refusals at its keys.
|
||||
foreign: Vec<(usize, CultCacheEnvelope)>,
|
||||
/// Whether the cache holds an image of its stores. While it is not `Pulled`, every write and removal is refused.
|
||||
load: LoadState,
|
||||
/// Why the last pull of the backing stores was refused; `None` once a pull succeeds. While set, every write and removal is refused.
|
||||
load_refused: Option<String>,
|
||||
}
|
||||
|
||||
/// What the cache knows of its stores. `Unpulled` is the state of a cache that has never pulled, or that gained a store
|
||||
/// since: its image lacks whatever the stores hold, and a batch would replace them with it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
enum LoadState {
|
||||
Unpulled,
|
||||
Pulled,
|
||||
/// Why the last pull was refused.
|
||||
Refused(String),
|
||||
}
|
||||
|
||||
/// A write or removal refused because the cache holds no image of its backing stores: its last load was refused, or it has
|
||||
/// not pulled since its stores were attached. Any write would persist an image that lacks the records it never read.
|
||||
/// Pull; for a refused load, first repair the store named by `reason` (which names a type, a key and a clause, never a
|
||||
/// payload).
|
||||
/// A write or removal refused because the cache's last load of its backing stores was refused. The cache holds no
|
||||
/// image of those stores, so any write would persist an image that lacks the records it never read. Repair the store
|
||||
/// named by `reason` (which names a type, a key and a clause, never a payload) and load again.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CultCacheLoadRefused {
|
||||
pub reason: String,
|
||||
|
|
@ -2950,7 +2953,7 @@ impl std::fmt::Display for CultCacheLoadRefused {
|
|||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"CultCache refuses writes and removals: it holds no image of its backing stores ({}); repair the store if it was refused and pull again",
|
||||
"CultCache refuses writes and removals: its load of the backing stores failed ({}); repair the store and load again",
|
||||
self.reason
|
||||
)
|
||||
}
|
||||
|
|
@ -3019,7 +3022,7 @@ impl CultCache {
|
|||
projectors: BTreeMap::new(),
|
||||
stores: Vec::new(),
|
||||
foreign: Vec::new(),
|
||||
load: LoadState::Unpulled,
|
||||
load_refused: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3125,10 +3128,6 @@ impl CultCache {
|
|||
store: Box::new(store),
|
||||
types,
|
||||
});
|
||||
let loaded_before = self.load.clone();
|
||||
if self.load == LoadState::Pulled {
|
||||
self.load = LoadState::Unpulled;
|
||||
}
|
||||
// A record's home cannot change after it is admitted.
|
||||
let candidate = self.stores.len() - 1;
|
||||
let held: BTreeSet<String> = self.entries.keys().map(|(type_id, _)| type_id.clone()).collect();
|
||||
|
|
@ -3147,7 +3146,6 @@ impl CultCache {
|
|||
self.describe_store(after)
|
||||
);
|
||||
self.stores.pop();
|
||||
self.load = loaded_before;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
|
@ -3166,27 +3164,21 @@ impl CultCache {
|
|||
self.entries = loaded;
|
||||
self.foreign = foreign;
|
||||
self.index_holders = holders;
|
||||
self.load = LoadState::Pulled;
|
||||
self.load_refused = None;
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => {
|
||||
self.load = LoadState::Refused(format!("{error:#}"));
|
||||
self.load_refused = Some(format!("{error:#}"));
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The one gate of every write and removal: a cache whose load was refused holds no image of its stores.
|
||||
/// A cache without stores has nothing to lose and needs no pull.
|
||||
fn require_loaded(&self) -> Result<()> {
|
||||
match &self.load {
|
||||
_ if self.stores.is_empty() => Ok(()),
|
||||
LoadState::Pulled => Ok(()),
|
||||
LoadState::Refused(reason) => Err(CultCacheLoadRefused { reason: reason.clone() }.into()),
|
||||
LoadState::Unpulled => Err(CultCacheLoadRefused {
|
||||
reason: "pull_all_backing_stores has not run since the stores were attached".to_string(),
|
||||
}
|
||||
.into()),
|
||||
match &self.load_refused {
|
||||
Some(reason) => Err(CultCacheLoadRefused { reason: reason.clone() }.into()),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3277,12 +3269,9 @@ impl CultCache {
|
|||
|
||||
/// Whether a write or removal of `(type_id, key)` would be admitted: false when the held payload carries slots past
|
||||
/// its registered type's declared slots, at any depth, or cannot be read by this build at all (the record is
|
||||
/// read-only), when the cache holds no image of its stores (see [`CultCacheLoadRefused`]), when a store carries a foreign record at the key, or when another registered type holds it. True for a
|
||||
/// read-only), when a store carries a foreign record at the key, or when another registered type holds it. True for a
|
||||
/// key nothing holds. An unregistered type is an error. Advisory: the write entries ask the same checks themselves.
|
||||
pub fn is_writable(&self, type_id: &str, key: &str) -> Result<bool> {
|
||||
if self.require_loaded().is_err() {
|
||||
return Ok(false);
|
||||
}
|
||||
if !matches!(self.held_finding(type_id, key)?, HeldFinding::Free) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
|
@ -3588,12 +3577,18 @@ impl CultCache {
|
|||
}
|
||||
let change = self.judge_unique(&staged.iter().collect::<Vec<_>>())?;
|
||||
if let Some(store_index) = store_index {
|
||||
// The home store is handed the staged records alone and applies them onto what it holds now.
|
||||
// A key staged twice lands as its last record, whatever the store.
|
||||
let landing: BTreeMap<_, _> = staged.iter().map(|entry| (entry_id(entry), entry.clone())).collect();
|
||||
// The home store receives its own records only: those it already holds in this cache plus the staged ones, and
|
||||
// the records it carries that no registered type claims: push_all replaces the store's records, so a foreign
|
||||
// record is laid back as the store read it.
|
||||
let snapshot: Vec<_> = next
|
||||
.values()
|
||||
.filter(|entry| self.resolve_route_index(&entry.r#type) == Some(store_index))
|
||||
.chain(self.foreign.iter().filter(|(home, _)| *home == store_index).map(|(_, entry)| entry))
|
||||
.cloned()
|
||||
.collect();
|
||||
self.stores[store_index]
|
||||
.store
|
||||
.push_all(&landing.into_values().collect::<Vec<_>>(), PushAllOptions::default())?;
|
||||
.push_all(&snapshot, PushAllOptions::default())?;
|
||||
}
|
||||
self.entries = next;
|
||||
change.apply(&mut self.index_holders);
|
||||
|
|
@ -4120,7 +4115,7 @@ struct DiskStore {
|
|||
}
|
||||
|
||||
impl DiskStore {
|
||||
/// Nothing at the path: an empty v1 store. A dangling link is something at the path and is refused, not read as this.
|
||||
/// A file that is gone, or a dangling link a whole-file replace is about to replace: an empty v1 store.
|
||||
fn absent() -> Self {
|
||||
Self { absent: true, format: STORE_FORMAT_V1, envelopes: Vec::new(), catalog: Vec::new() }
|
||||
}
|
||||
|
|
@ -4304,6 +4299,12 @@ fn release_lock<T>(lock: File, result: Result<T>) -> Result<T> {
|
|||
result
|
||||
}
|
||||
|
||||
fn io_not_found(error: &anyhow::Error) -> bool {
|
||||
error
|
||||
.chain()
|
||||
.any(|cause| cause.downcast_ref::<std::io::Error>().is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound))
|
||||
}
|
||||
|
||||
/// True only when nothing is at `path`; a symbolic link, dangling or not, is something.
|
||||
/// A path that cannot be inspected (a file where a directory should be, no permission)
|
||||
/// is an error, never absent.
|
||||
|
|
@ -4723,7 +4724,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
records(&mut cache)
|
||||
}
|
||||
|
||||
|
|
@ -4965,7 +4965,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&routed), [Settings::TYPE])?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
let (staged, _) = cache.prepare_entry("other", &settings_one())?;
|
||||
|
|
@ -5067,7 +5066,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
|
||||
cache.put("shared", &settings_one())?;
|
||||
let before = std::fs::read(&store_path)?;
|
||||
|
|
@ -5100,7 +5098,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put("kept", &settings_one())?;
|
||||
let before = std::fs::read(&store_path)?;
|
||||
|
||||
|
|
@ -5209,7 +5206,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
|
||||
let (settings, _) = cache.prepare_entry(
|
||||
"app",
|
||||
|
|
@ -5374,12 +5370,6 @@ mod tests {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn io_not_found(error: &anyhow::Error) -> bool {
|
||||
error
|
||||
.chain()
|
||||
.any(|cause| cause.downcast_ref::<std::io::Error>().is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound))
|
||||
}
|
||||
|
||||
fn test_envelope(r#type: &str, key: &str, payload: &[u8]) -> CultCacheEnvelope {
|
||||
CultCacheEnvelope {
|
||||
key: key.to_string(),
|
||||
|
|
@ -5922,7 +5912,6 @@ mod tests {
|
|||
inner: SingleFileMessagePackBackingStore::new(&store_path),
|
||||
refuse_batch: refusal.clone(),
|
||||
})?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put(
|
||||
"app",
|
||||
&Settings {
|
||||
|
|
@ -5963,7 +5952,6 @@ mod tests {
|
|||
["settings"],
|
||||
)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
|
||||
cache.put(
|
||||
"app",
|
||||
|
|
@ -5992,7 +5980,6 @@ mod tests {
|
|||
let temp = tempfile::tempdir()?;
|
||||
let mut cache = CultCache::new();
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("a.cc")))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
let error = cache
|
||||
.add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("b.cc")))
|
||||
.unwrap_err();
|
||||
|
|
@ -6006,7 +5993,6 @@ mod tests {
|
|||
let temp = tempfile::tempdir()?;
|
||||
let mut cache = CultCache::new();
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("a.cc")), ["settings"])?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
let error = cache
|
||||
.add_backing_store(
|
||||
SingleFileMessagePackBackingStore::new(temp.path().join("b.cc")),
|
||||
|
|
@ -6016,7 +6002,6 @@ mod tests {
|
|||
assert!(error.to_string().contains("already routed"), "{error}");
|
||||
// A refused registration claims nothing, so "note" is still free.
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("c.cc")), ["note"])?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
assert_eq!(cache.stores.len(), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -6029,7 +6014,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put("app", &Settings { theme: "v1-generic".to_string(), retries: 1 })?;
|
||||
|
||||
let error = cache
|
||||
|
|
@ -6059,17 +6043,15 @@ mod tests {
|
|||
let mut writer = CultCache::new();
|
||||
writer.register_entry_type::<Settings>()?;
|
||||
writer.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
writer.pull_all_backing_stores()?;
|
||||
writer.put("app", &Settings { theme: "stray".to_string(), retries: 0 })?;
|
||||
let generic_bytes = fs::read(&generic_path)?;
|
||||
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"])?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
// The cache already holds a record, in its home store, before the refused load.
|
||||
cache.put("held", &Settings { theme: "home".to_string(), retries: 0 })?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
let before = cache.snapshot();
|
||||
let settings_bytes = fs::read(&settings_path)?;
|
||||
let error = cache.pull_all_backing_stores().unwrap_err();
|
||||
|
|
@ -6093,9 +6075,7 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), [Settings::TYPE])?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put("app", &Settings { theme: "before".to_string(), retries: 1 })?;
|
||||
let before = cache.snapshot();
|
||||
|
||||
|
|
@ -6137,7 +6117,6 @@ mod tests {
|
|||
let mut homeless = CultCache::new();
|
||||
homeless.register_registry(TestEntries)?;
|
||||
homeless.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("settings.cc")), ["settings"])?;
|
||||
homeless.pull_all_backing_stores()?;
|
||||
homeless.put("held", &Settings { theme: "held".to_string(), retries: 0 })?;
|
||||
let before = homeless.snapshot();
|
||||
let error = homeless.load_soa(table).unwrap_err();
|
||||
|
|
@ -6154,9 +6133,7 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&settings_path), ["settings"])?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&generic_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put("memo", &Note { title: "generic".to_string(), body: "home".to_string() })?;
|
||||
cache.put("old", &Settings { theme: "kept".to_string(), retries: 0 })?;
|
||||
|
||||
|
|
@ -6187,7 +6164,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put("app", &Settings { theme: "before".to_string(), retries: 1 })?;
|
||||
let before = cache.snapshot();
|
||||
let store_bytes = fs::read(&store_path)?;
|
||||
|
|
@ -6227,7 +6203,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
|
||||
let updated = cache.update::<Settings, _>("app", |current| {
|
||||
let mut current = current.unwrap_or(Settings {
|
||||
|
|
@ -6250,7 +6225,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put(
|
||||
"app",
|
||||
&Settings {
|
||||
|
|
@ -6314,7 +6288,6 @@ mod tests {
|
|||
let mut origin = CultCache::new();
|
||||
origin.register_entry_type::<Settings>()?;
|
||||
origin.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&origin_store))?;
|
||||
origin.pull_all_backing_stores()?;
|
||||
origin.put(
|
||||
"app",
|
||||
&Settings {
|
||||
|
|
@ -6328,7 +6301,6 @@ mod tests {
|
|||
let mut target = CultCache::new();
|
||||
target.register_entry_type::<Settings>()?;
|
||||
target.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&target_store))?;
|
||||
target.pull_all_backing_stores()?;
|
||||
let applied = target.put_envelope::<Settings>(envelope.clone())?;
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -6362,7 +6334,6 @@ mod tests {
|
|||
cache.register_entry_type::<Settings>()?;
|
||||
cache.register_entry_type::<Note>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put_raw_envelope(envelope.clone())?;
|
||||
|
||||
assert_eq!(cache.snapshot(), vec![envelope.clone()]);
|
||||
|
|
@ -6405,7 +6376,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
let tailed = CultCacheEnvelope {
|
||||
key: "app".to_string(),
|
||||
r#type: Settings::TYPE.to_string(),
|
||||
|
|
@ -6434,7 +6404,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put(
|
||||
"app",
|
||||
&Settings {
|
||||
|
|
@ -6472,7 +6441,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put(
|
||||
"settings:a",
|
||||
&Settings {
|
||||
|
|
@ -6555,7 +6523,6 @@ mod tests {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Settings>()?;
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&store_path))?;
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put(
|
||||
"primary",
|
||||
&Settings {
|
||||
|
|
@ -7960,17 +7927,15 @@ mod tests {
|
|||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_push_all_through_a_dangling_link_is_refused_like_push_and_writes_nothing() -> Result<()> {
|
||||
fn r3_decides_resolve_or_refuse_a_push_all_through_a_dangling_link_replaces_the_link() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let volume = temp.path().join("volume");
|
||||
fs::create_dir(&volume)?;
|
||||
let link = temp.path().join("store.cc");
|
||||
std::os::unix::fs::symlink(volume.join("store.cc"), &link)?;
|
||||
|
||||
let mut store = SingleFileMessagePackBackingStore::new(&link);
|
||||
assert!(store.push_all(&[snapshot_envelope("a", b"1")], PushAllOptions::default()).is_err());
|
||||
assert!(store.push(&snapshot_envelope("a", b"1")).is_err());
|
||||
assert!(fs::symlink_metadata(&link)?.file_type().is_symlink(), "the link is left as it was");
|
||||
SingleFileMessagePackBackingStore::new(&link).push_all(&[snapshot_envelope("a", b"1")], PushAllOptions::default())?;
|
||||
assert!(fs::symlink_metadata(&link)?.file_type().is_file(), "the link was replaced by a regular file");
|
||||
assert_eq!(fs::read_dir(&volume)?.count(), 0, "nothing reached the link's volume");
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -8086,176 +8051,4 @@ mod tests {
|
|||
assert_eq!(fs::read(&path)?, before);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A store already holding `Settings:"held"`, then a fresh cache attached to it that has not pulled.
|
||||
fn unpulled_cache_over_populated_stores() -> Result<(tempfile::TempDir, Vec<CultCache>, Vec<PathBuf>)> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let single = temp.path().join("single.cc");
|
||||
let redb = temp.path().join("redb.db");
|
||||
let held = Settings { theme: "held".to_string(), retries: 1 };
|
||||
let mut seeder = CultCache::new();
|
||||
seeder.register_registry(TestEntries)?;
|
||||
seeder.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&single))?;
|
||||
seeder.pull_all_backing_stores()?;
|
||||
seeder.put("held", &held)?;
|
||||
let mut seeder = CultCache::new();
|
||||
seeder.register_registry(TestEntries)?;
|
||||
seeder.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb)?)?;
|
||||
seeder.pull_all_backing_stores()?;
|
||||
seeder.put("held", &held)?;
|
||||
|
||||
let mut over_single = CultCache::new();
|
||||
over_single.register_registry(TestEntries)?;
|
||||
over_single.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&single))?;
|
||||
let mut over_redb = CultCache::new();
|
||||
over_redb.register_registry(TestEntries)?;
|
||||
over_redb.add_generic_backing_store(RedbMessagePackBackingStore::new(&redb)?)?;
|
||||
Ok((temp, vec![over_single, over_redb], vec![single, redb]))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cache_that_never_pulled_refuses_every_write_and_removal() -> Result<()> {
|
||||
let (_temp, caches, _paths) = unpulled_cache_over_populated_stores()?;
|
||||
for mut cache in caches {
|
||||
let fresh = Note { title: "t".to_string(), body: "b".to_string() };
|
||||
let (staged, _) = cache.prepare_entry("fresh", &fresh)?;
|
||||
for error in [
|
||||
cache.put_prepared_batch(vec![staged.clone()]).unwrap_err(),
|
||||
cache.put("fresh", &fresh).unwrap_err(),
|
||||
cache.put_raw_envelope(staged.clone()).unwrap_err(),
|
||||
cache.delete::<Settings>("held").unwrap_err(),
|
||||
] {
|
||||
assert!(error.downcast_ref::<CultCacheLoadRefused>().is_some(), "{error:#}");
|
||||
}
|
||||
assert!(cache.snapshot().is_empty());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_batch_from_a_cache_that_never_pulled_cannot_wipe_the_store() -> Result<()> {
|
||||
let (_temp, caches, paths) = unpulled_cache_over_populated_stores()?;
|
||||
for (mut cache, path) in caches.into_iter().zip(paths) {
|
||||
let (staged, _) = cache.prepare_entry("fresh", &Note { title: "t".to_string(), body: "b".to_string() })?;
|
||||
let _ = cache.put_prepared_batch(vec![staged]);
|
||||
let mut reader = CultCache::new();
|
||||
reader.register_registry(TestEntries)?;
|
||||
if path.extension().is_some_and(|extension| extension == "db") {
|
||||
reader.add_generic_backing_store(RedbMessagePackBackingStore::new(&path)?)?;
|
||||
reader.pull_all_backing_stores()?;
|
||||
} else {
|
||||
reader.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?;
|
||||
reader.pull_all_backing_stores()?;
|
||||
}
|
||||
reader.pull_all_backing_stores()?;
|
||||
assert_eq!(reader.get_required::<Settings>("held")?.theme, "held", "{path:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pulling_lifts_the_gate_and_a_store_attached_later_closes_it_again() -> Result<()> {
|
||||
let (temp, mut caches, _paths) = unpulled_cache_over_populated_stores()?;
|
||||
let mut cache = caches.remove(0);
|
||||
cache.pull_all_backing_stores()?;
|
||||
cache.put("fresh", &Note { title: "t".to_string(), body: "b".to_string() })?;
|
||||
assert_eq!(cache.get_required::<Settings>("held")?.theme, "held");
|
||||
|
||||
let mut routed = CultCache::new();
|
||||
routed.register_registry(TestEntries)?;
|
||||
let settings = Settings { theme: "t".to_string(), retries: 0 };
|
||||
routed.add_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("routed.cc")), [Settings::TYPE])?;
|
||||
routed.pull_all_backing_stores()?;
|
||||
routed.put("fresh", &settings)?;
|
||||
routed.add_generic_backing_store(SingleFileMessagePackBackingStore::new(temp.path().join("other.cc")))?;
|
||||
assert!(routed.put("again", &settings).unwrap_err().downcast_ref::<CultCacheLoadRefused>().is_some());
|
||||
routed.pull_all_backing_stores()?;
|
||||
routed.put("again", &settings)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_cache_batch_keeps_the_records_another_writer_added() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let single = temp.path().join("single.cc");
|
||||
let redb = temp.path().join("redb.db");
|
||||
let open = |path: &Path| -> Result<CultCache> {
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
if path.extension().is_some_and(|extension| extension == "db") {
|
||||
cache.add_generic_backing_store(RedbMessagePackBackingStore::new(path)?)?;
|
||||
} else {
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(path))?;
|
||||
}
|
||||
cache.pull_all_backing_stores()?;
|
||||
Ok(cache)
|
||||
};
|
||||
for path in [&single, &redb] {
|
||||
let mut seed = open(path)?;
|
||||
seed.put("held", &Settings { theme: "held".to_string(), retries: 1 })?;
|
||||
let mut stale = open(path)?;
|
||||
let mut other = open(path)?;
|
||||
other.put("held", &Settings { theme: "other-newer".to_string(), retries: 3 })?;
|
||||
other.put("added", &Settings { theme: "other".to_string(), retries: 2 })?;
|
||||
let (staged, _) = stale.prepare_entry("mine", &Note { title: "t".to_string(), body: "b".to_string() })?;
|
||||
stale.put_prepared_batch(vec![staged])?;
|
||||
let reader = open(path)?;
|
||||
assert_eq!(reader.get_required::<Settings>("added")?.theme, "other", "{path:?}");
|
||||
assert_eq!(reader.get_required::<Settings>("held")?.theme, "other-newer", "a record another writer updated is not reverted");
|
||||
assert_eq!(reader.get_required::<Note>("mine")?.title, "t", "{path:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_store_merges_push_all_and_lands_the_last_of_a_repeated_key() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let stores: Vec<Box<dyn CacheBackingStore>> = vec![
|
||||
Box::new(SingleFileMessagePackBackingStore::new(temp.path().join("single.cc"))),
|
||||
Box::new(RedbMessagePackBackingStore::new(temp.path().join("redb.db"))?),
|
||||
Box::new(OwnedRedbMessagePackBackingStore::new(temp.path().join("owned.db"))?),
|
||||
];
|
||||
for mut store in stores {
|
||||
store.push(&test_envelope("pushall.merge", "kept", b"kept"))?;
|
||||
store.push_all(&[test_envelope("pushall.merge", "new", b"new")], PushAllOptions::default())?;
|
||||
let keys: Vec<_> = store.pull_all()?.into_iter().map(|entry| entry.key).collect();
|
||||
assert_eq!(keys, ["kept", "new"]);
|
||||
}
|
||||
// A batch naming a key twice lands the last, in the single-file and the redb stores alike.
|
||||
for path in [temp.path().join("dup.cc"), temp.path().join("dup.db")] {
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_registry(TestEntries)?;
|
||||
if path.extension().is_some_and(|extension| extension == "db") {
|
||||
cache.add_generic_backing_store(RedbMessagePackBackingStore::new(&path)?)?;
|
||||
} else {
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path))?;
|
||||
}
|
||||
cache.pull_all_backing_stores()?;
|
||||
let (first, _) = cache.prepare_entry("k", &Settings { theme: "first".to_string(), retries: 1 })?;
|
||||
let (last, _) = cache.prepare_entry("k", &Settings { theme: "last".to_string(), retries: 2 })?;
|
||||
cache.put_prepared_batch(vec![first, last])?;
|
||||
assert_eq!(cache.get_required::<Settings>("k")?.theme, "last");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_all_replaces_a_record_held_under_another_schema_id_as_push_does() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let mut store = SingleFileMessagePackBackingStore::new(temp.path().join("store.cc"));
|
||||
let mut held = snapshot_envelope("k", b"old");
|
||||
held.schema_id = Some("legacy.id".to_string());
|
||||
store.push(&held)?;
|
||||
let mut replacement = snapshot_envelope("k", b"new");
|
||||
replacement.schema_id = None;
|
||||
let mut by_push = store.clone();
|
||||
by_push.push(&replacement)?;
|
||||
assert_eq!(store.pull_all()?.len(), 1);
|
||||
store.push(&held)?;
|
||||
store.push_all(&[replacement.clone()], PushAllOptions::default())?;
|
||||
let held_now = store.pull_all()?;
|
||||
assert_eq!(held_now.len(), 1, "one record under the key");
|
||||
assert_eq!(held_now[0].payload, replacement.payload);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -149,7 +149,6 @@ fn every_read_only_vector_decodes_marked_and_is_not_writable() {
|
|||
let mut cache = registered();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc"))).unwrap();
|
||||
cache.pull_all_backing_stores().unwrap();
|
||||
cache.put_raw_envelope(envelope("k", payload.clone())).unwrap();
|
||||
assert_eq!(cache.get::<UnionSurface>("k").unwrap(), Some(constructed()), "{file}");
|
||||
assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap(), "{file}");
|
||||
|
|
@ -471,7 +470,6 @@ fn is_writable_is_false_for_a_type_with_no_home_store() {
|
|||
let mut cache = registered();
|
||||
// One store, routed to SlotUnit only: UnionSurface has no home.
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(dir.path().join("s.cc")), [SlotUnit::TYPE]).unwrap();
|
||||
cache.pull_all_backing_stores().unwrap();
|
||||
assert!(cache.put("k", &constructed()).is_err());
|
||||
assert!(!cache.is_writable(UnionSurface::TYPE, "k").unwrap());
|
||||
assert!(cache.is_writable(SlotUnit::TYPE, "k").unwrap());
|
||||
|
|
|
|||
|
|
@ -619,23 +619,6 @@ fn depth_is_refused_at_load() {
|
|||
assert!(cache.snapshot().is_empty());
|
||||
}
|
||||
|
||||
// load_soa is a load too: a table holding a payload nested past the bound is refused with the typed depth refusal, the
|
||||
// prior view is kept, and a tail behind the depth is still the depth refusal.
|
||||
#[test]
|
||||
fn load_soa_refuses_a_payload_nested_past_the_bound() {
|
||||
let mut deep = vec![0x91u8; MAX_PAYLOAD_DEPTH + 1];
|
||||
deep.push(0xc0);
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<UnionSurface>().unwrap();
|
||||
cache.load_soa(CultCacheSoaTable::from_envelopes(vec![envelope("kept", &vector("union-surface.msgpack"))])).unwrap();
|
||||
let before = cache.snapshot();
|
||||
for payload in [deep.clone(), [deep.clone(), vec![0xc0]].concat()] {
|
||||
let error = cache.load_soa(CultCacheSoaTable::from_envelopes(vec![envelope("k", &payload)])).unwrap_err();
|
||||
assert_eq!(error.downcast_ref::<PayloadTooDeep>(), Some(&TOO_DEEP), "{error:#}");
|
||||
assert_eq!(cache.snapshot(), before, "a refused load keeps the prior view");
|
||||
}
|
||||
}
|
||||
|
||||
// A cache whose load was refused refuses every write and removal and leaves the store as it was: a
|
||||
// later write would otherwise persist the empty image over the records it never read.
|
||||
#[test]
|
||||
|
|
@ -735,7 +718,6 @@ fn write_refused_past_limit() {
|
|||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<UnionSurface>().unwrap();
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||
cache.pull_all_backing_stores().unwrap();
|
||||
|
||||
let at_limit = wrap_chain(126, boxed("0000000000f0", vec![]));
|
||||
assert_eq!(cache.put("at", &at_limit).unwrap(), at_limit);
|
||||
|
|
@ -819,7 +801,6 @@ fn the_raw_doors_hold_the_bound_for_arms_maps_and_arrays() {
|
|||
cache.register_entry_type::<UnionSurface>().unwrap();
|
||||
cache.register_entry_type::<MapDoc>().unwrap();
|
||||
cache.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&path)).unwrap();
|
||||
cache.pull_all_backing_stores().unwrap();
|
||||
cache.put_raw_envelope(raw(type_name, "at-raw", at.clone())).unwrap();
|
||||
cache.put_prepared_batch(vec![raw(type_name, "at-batch", at.clone())]).unwrap();
|
||||
let stored = cache.snapshot_envelopes();
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cultcache_rs::{
|
||||
CultCache, CultCacheEnvelope, CultCacheLoadRefused, CultCacheSoaTable, DatabaseEntry, SingleFileMessagePackBackingStore,
|
||||
CultCache, CultCacheEnvelope, CultCacheSoaTable, DatabaseEntry, SingleFileMessagePackBackingStore,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
|
|
@ -419,54 +419,6 @@ fn a_load_of_a_duplicate_image_is_refused_and_keeps_the_prior_view() {
|
|||
soa.put("item:y", &unique("K")).unwrap();
|
||||
}
|
||||
|
||||
// A pull refused for a duplicate unique-index value, or for one key held under two types in two stores, closes the same
|
||||
// write gate a payload refusal does: the cache holds no image of its stores, so a write would lay the empty image over them.
|
||||
#[test]
|
||||
fn a_pull_refused_for_a_duplicate_index_or_a_retype_closes_every_write() {
|
||||
fn assert_gated(cache: &mut CultCache, path: &Path, door_label: &str) {
|
||||
let before = std::fs::read(path).unwrap();
|
||||
let errors = [
|
||||
("put", cache.put("item:new", &unique("N")).unwrap_err()),
|
||||
("put_raw_envelope", cache.put_raw_envelope(cache.prepare_entry("item:new", &unique("N")).unwrap().0).unwrap_err()),
|
||||
("delete", cache.delete::<Unique>("item:keep").unwrap_err()),
|
||||
];
|
||||
for (door, error) in errors {
|
||||
assert!(error.downcast_ref::<CultCacheLoadRefused>().is_some(), "{door_label} {door}: {error:#}");
|
||||
}
|
||||
assert_eq!(std::fs::read(path).unwrap(), before, "{door_label}: the store changed");
|
||||
}
|
||||
|
||||
// Duplicate index value found in the stored image.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("dup.cc");
|
||||
let mut cache = cache_at(&path);
|
||||
cache.put("item:keep", &unique("K")).unwrap();
|
||||
let mut twin = twin_cache_at(&path);
|
||||
twin.put("item:b", &Twin { sku: "S".into(), category: "c".into() }).unwrap();
|
||||
twin.put("item:a", &Twin { sku: "S".into(), category: "c".into() }).unwrap();
|
||||
assert!(error_text(cache.pull_all_backing_stores()).contains("both hold unique index 'Sku'"));
|
||||
assert_gated(&mut cache, &path, "duplicate index");
|
||||
|
||||
// One key held under two types by two stores. The cache pulled cleanly first, so only the refused pull can close the gate.
|
||||
let (first, second) = (dir.path().join("first.cc"), dir.path().join("second.cc"));
|
||||
let mut cache = CultCache::new();
|
||||
cache.register_entry_type::<Plain>().unwrap();
|
||||
cache.register_entry_type::<Other>().unwrap();
|
||||
cache.register_entry_type::<Unique>().unwrap();
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&first), [Plain::TYPE]).unwrap();
|
||||
cache.add_backing_store(SingleFileMessagePackBackingStore::new(&second), [Other::TYPE]).unwrap();
|
||||
cache.pull_all_backing_stores().unwrap();
|
||||
cache.put("k", &Plain { value: "p".into() }).unwrap();
|
||||
let mut seed = CultCache::new();
|
||||
seed.register_entry_type::<Other>().unwrap();
|
||||
seed.add_generic_backing_store(SingleFileMessagePackBackingStore::new(&second)).unwrap();
|
||||
seed.pull_all_backing_stores().unwrap();
|
||||
seed.put("k", &Other { sku: "o".into() }).unwrap();
|
||||
assert!(error_text(cache.pull_all_backing_stores()).contains("\"k\""));
|
||||
assert_gated(&mut cache, &first, "retype");
|
||||
assert_eq!(cache.get::<Plain>("k").unwrap().unwrap().value, "p", "a refused pull keeps the prior view");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_multi_index_store_with_a_duplicate_on_the_second_alias_refuses_to_open() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following
|
|||
|
||||
### Breaking
|
||||
|
||||
- `SingleFileMessagePackBackingStore.pushAll` through a dangling symbolic link at the store path is refused with `ENOENT`, as `push` and `pullAll` are, and writes nothing; it used to replace the link with a new store. The reader owns the verdict on what is at the path.
|
||||
- `SingleFileMessagePackBackingStore.pushAll` writes the given entries onto the store as it is under the lock, as `push` does: a record under the same key is replaced and no other is removed. It used to replace the whole file with the entries, so `CultMesh.flush()` from a cache that held less than the store (a node created with `pullOnStart: false`, or one whose image was stale) wiped the records it never read. A caller that relied on `pushAll` to reset a store removes the file first. A soft flush is unchanged. `CultCache.storedSnapshot()` is the image as the stores hold it (a global loaded from a legacy key under that key) and is what `CultMesh.flush` persists, so a flush no longer leaves such a global under two keys; `snapshot()` is unchanged.
|
||||
- A single-file store marked `cultcache.store.v3` (it holds an element id, written by the C# runtime) is read, and a rewrite keeps the mark and the ids; this runtime does not decide when a document carries ids. A runtime from before this change refuses a v3 store by name, so rolling back to an older release after a store has been written as v3 is one-way: there is no carry-back, so restore from a copy taken before upgrading.
|
||||
- `indexes` are unique, as C# `[CultIndex]`: a write or a load that leaves two records of a type holding one non-blank
|
||||
value under one alias is refused naming both keys and the alias, and writes nothing. Index text is the C# canonical
|
||||
|
|
@ -22,16 +20,13 @@ All notable changes to `@gamecult/cultcache-ts` are recorded here, following
|
|||
- A store whose canonical-schema catalog slot is not a string is refused as `StoreUnreadableError`.
|
||||
- The cache holds one record per key (variants:ruling:cache-refuses-retype). A `put` or `putEnvelope` that lands a second type under a key the cache holds, or under a key a store carries foreign, is refused with `SchemaConflictError` naming the key, even with no store (a memory-only cache used to hold both types at the key). A type that owns the held record's schema id (as for the store) replaces it, unless the held record is in a store other than the written type's home: that rename is refused with `SchemaConflictError` (remove the record first). Two stores holding one key, registered or foreign, refuse the load and admit nothing. A store's `delete` removes the record whose stored schema id the removal's type owns, whatever the arrived record's `type` string.
|
||||
- A store holding a record no registered type claims now loads (it used to throw "No schema is registered..."): the cache does not serve the record, lists it in `foreignRecords()`, keeps its bytes through every write, and refuses a `put` or a `delete` at its key with `SchemaConflictError` (variants:ruling:unheld-removal-refused-at-commit). Registering a type that claims its schema id serves it at the next `pullAllBackingStores`.
|
||||
- A record whose payload carries a slot past its reader's declared slots, at any depth (the record, a nested record, a list element, a union arm, a unit arm), is now held read-only: it loads and reads, and every `put`, `putEnvelope`, `update` or `delete` of its key is refused with `SchemaConflictError` before a store is touched (cultlib-gaps:ruling:extra-trailing-slots, as the C# reference). A load used to skip the extra slots silently and a later write shed them. The refusal names the schema, the key, the type that was read with extra slots and both slot counts, never a slot value, and carries them as `SchemaConflictError.unreadSlots`. An untouched record keeps its bytes through writes of other keys. Missing trailing slots keep their declared defaults and mark nothing. A `put` never marks (the value is encoded by this runtime); a `putEnvelope` of a payload carrying extra slots lands its bytes and holds the record read-only. A mark at depth 0 is consented to only when the record arrived under a schema id other than the reader's own that the reader lists compatible, and that id's catalog entry lists those slots (a writer that adds a slot and keeps the reader's id is never consented to) (`unreadSlots.explained`); every other mark refuses whatever is declared. New: `CultCache.isWritable(definition, key)` (live, advisory), `CultCache.unreadSlots(definition, key)` and the exported `CultCacheUnreadSlots`. Documents with no slot codec (a plain map, a user formatter) never mark. A `delete` is by key: a type that owns the schema id of the record held at the key as another type removes that one record (one store call, the cache dropped once the store has removed it), and a type that does not own it is refused with `SchemaConflictError`, as a `put` is; `isWritable` answers both. Implied semver: a minor bump (0.16.0) while the package is 0.x; a major one at 1.0 and after.
|
||||
|
||||
### Added
|
||||
|
||||
- `checkPayloadDepth` returns the offset where the first value ends, or `undefined` when the payload ends early (a header, a container or a str/bin/scalar body cut short; it was `void`); it still refuses only depth. The bytes-after refusal is internal (`checkPayloadIsOneValue`, not exported); no new public name.
|
||||
- `CultCache.foreignRecords()`: the `{key, schemaId, schemaName, storedAt}` of every record the last pull carried and no registered type claims, sorted by key ordinal.
|
||||
|
||||
### Changed
|
||||
|
||||
- A document payload is exactly one MessagePack value (cultlib-gaps:ruling:trailing-bytes-refuse). `decodeDocumentPayload`, the one reader (store loads, `put`/`putEnvelope`, CultNet replication, the provider command), and the inspector refuse bytes after the first value with `document type "T" payload has N bytes after its first MessagePack value.`, counted from the walk's end and never walked or echoed; a payload that ends early stays the decoder's to refuse. The refused set is unchanged; only the text changed (it was `@msgpack`'s `RangeError` "Extra N of M byte(s)" for a plain document and the codec reader's "bytes after its value" for a codec document, and the inspector's preview no longer shows a tail), so code matching either text must match this one. `readWire` reads one value and no longer checks what follows. Stated limit: a definition with its own `formatter` is not walked, so whether its payload is one value is the formatter's. Implied bump: 0.16.0 (pre-1.0 minor, with the Breaking entries above).
|
||||
- Opening a legacy (pre-v1) store is a pure read: payloads that were not bytes are normalized in memory and the
|
||||
file is left as it was, so a codec that refuses the content cannot lose the old bytes. The next push, delete or
|
||||
flush rewrites the file as v1, merging what the open read.
|
||||
|
|
|
|||
|
|
@ -49,9 +49,7 @@ through `decodeDocumentPayload`, and when you need the bytes themselves (hashing
|
|||
write) take them with `documentPayloadBytes`, the sanctioned escape.
|
||||
|
||||
- `decodeDocumentPayload(definition, payload)` reads a payload through the definition's formatter,
|
||||
codec or plain map and then its schema. The definition is required. A payload is exactly one
|
||||
MessagePack value: bytes after the first are refused by count, naming the type, and never walked or
|
||||
echoed (a payload that ends early is the decoder's to refuse).
|
||||
codec or plain map and then its schema. The definition is required.
|
||||
- `encodeDocumentPayload(definition, value)` is the writer: the schema first, then the formatter,
|
||||
codec or plain map.
|
||||
- `documentPayloadFromBytes(bytes)` is where bytes enter (a record you validated, a store read, a
|
||||
|
|
@ -174,7 +172,7 @@ const settings = cache.getRequiredGlobal(settingsDocument);
|
|||
- `updateGlobal(...)`
|
||||
- `delete(...)`
|
||||
- `deleteGlobal(...)`
|
||||
- `snapshot()` (the logical image) and `storedSnapshot()` (the image as the stores hold it, for a whole flush)
|
||||
- `snapshot()`
|
||||
- `getEnvelope(...)`
|
||||
- `getRequiredEnvelope(...)`
|
||||
- `getGlobalEnvelope(...)`
|
||||
|
|
@ -329,7 +327,7 @@ If you do not supply one, `CultCacheTS` uses MessagePack plus Zod validation by
|
|||
|
||||
The 256-container nesting bound (`MAX_PAYLOAD_DEPTH`, refused with `CultCachePayloadDepthError`) covers the MessagePack the
|
||||
library itself writes: the default format and a codec. A payload your formatter writes is opaque bytes in any format, so the
|
||||
library does not walk it as MessagePack; its format, its bounds and whether it is one value are your formatter's.
|
||||
library does not walk it as MessagePack; its format and its bounds are your formatter's.
|
||||
|
||||
Zod is no longer mandatory. Any schema object with a `parse(...)` function will
|
||||
do, which means generated Ajv-backed JSON-schema contracts can feed the cache
|
||||
|
|
@ -373,11 +371,8 @@ const sceneDocument = defineDocumentType({
|
|||
});
|
||||
```
|
||||
|
||||
- A slot is `[name, codec, default?]`. A gap is written nil and skipped on read; an extra trailing slot is read past and
|
||||
marks the record read-only (every write or removal of its key is refused with `SchemaConflictError`, `cache.isWritable`
|
||||
asks first). The one writable case is a depth-0 extra slot of a record that arrived under a schema id other than the reader's own, which the
|
||||
reader lists in `compatibleSchemaIds`, whose catalog entry lists those slots: declaring that id consents to the loss, and a write restamps
|
||||
the record. A nested slot, or a writer that keeps the reader's own id, is never consented to. A slot missing from the payload (or nil, when a default is declared) takes its default, and is refused without one.
|
||||
- A slot is `[name, codec, default?]`. A gap is written nil and skipped on read; an extra trailing slot is skipped; a
|
||||
slot missing from the payload (or nil, when a default is declared) takes its default, and is refused without one.
|
||||
- A union refuses an undeclared arm key on read, naming the union, the key and the declared keys. Nothing falls back
|
||||
to a default arm. An arm with no slots is `codecs.unit` and is written `[key, []]`.
|
||||
- A refusal names the union, the field and the slot; it never echoes the value it refused.
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ export type {
|
|||
CultCacheIndexScalar,
|
||||
InferCultCacheSchemaValue,
|
||||
CultCacheStoreRegistration,
|
||||
CultCacheUnreadSlots,
|
||||
PushAllOptions,
|
||||
} from "./types";
|
||||
export type {
|
||||
|
|
|
|||
|
|
@ -198,66 +198,6 @@ type SlotSpec<TValue, TName extends string = string> =
|
|||
|
||||
type SlotsSpec<TValue> = Readonly<Record<number, { [K in keyof TValue & string]: SlotSpec<TValue[K], K> }[keyof TValue & string]>>;
|
||||
|
||||
// The slots a payload decode read past, noted by `slots` and the unit arm while the one decode entry (decodeNoting) has a scope
|
||||
// open, and read out by that entry. Module-scoped as the reference's is thread-bound: a decode is synchronous, and a decode
|
||||
// outside the entry (a codec called directly, a CultMesh read) notes nothing. No codec signature carries it.
|
||||
export interface NotedSlots {
|
||||
/** The first type read with extra slots, outermost first. */
|
||||
readonly typeName: string;
|
||||
readonly slotCount: number;
|
||||
readonly declaredCount: number;
|
||||
readonly occurrences: number;
|
||||
/** True when every noted object was the document's own: a union adds no depth, each record or arm inside a field adds one. */
|
||||
readonly depth0Only: boolean;
|
||||
}
|
||||
|
||||
interface SlotScope {
|
||||
depth: number;
|
||||
found?: { typeName: string; slotCount: number; declaredCount: number; occurrences: number; depth0Only: boolean };
|
||||
}
|
||||
|
||||
let scope: SlotScope | undefined;
|
||||
|
||||
/** The one decode entry: runs the decode with a scope open and returns its value with what it noted. */
|
||||
export function decodeNoting<T>(decode: () => T): { value: T; unreadSlots?: NotedSlots } {
|
||||
const outer = scope;
|
||||
const mine: SlotScope = { depth: 0 };
|
||||
scope = mine;
|
||||
try {
|
||||
const value = decode();
|
||||
return mine.found ? { value, unreadSlots: mine.found } : { value };
|
||||
} finally {
|
||||
scope = outer;
|
||||
}
|
||||
}
|
||||
|
||||
function noteExtraSlots(typeName: string, slotCount: number, declaredCount: number): void {
|
||||
if (!scope) return;
|
||||
const found = scope.found;
|
||||
if (found) {
|
||||
found.occurrences += 1;
|
||||
found.depth0Only &&= scope.depth === 0;
|
||||
} else {
|
||||
scope.found = { typeName, slotCount, declaredCount, occurrences: 1, depth0Only: scope.depth === 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// What a record inside a field reads at: one deeper than the record holding the field.
|
||||
function nested<T>(read: () => T): T {
|
||||
if (!scope) return read();
|
||||
scope.depth += 1;
|
||||
try {
|
||||
return read();
|
||||
} finally {
|
||||
scope.depth -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// The reference's type name for a mark: the type, not its namespace or the type it is nested in.
|
||||
function simpleTypeName(typeName: string | undefined): string {
|
||||
return typeName === undefined ? "record" : typeName.slice(Math.max(typeName.lastIndexOf("."), typeName.lastIndexOf("+")) + 1);
|
||||
}
|
||||
|
||||
const INT32_MIN = -(2 ** 31);
|
||||
const INT32_MAX = 2 ** 31 - 1;
|
||||
|
||||
|
|
@ -292,7 +232,7 @@ function pascalCase(name: string): string {
|
|||
}
|
||||
|
||||
// A positional record: slot `key` holds field `name`. Gaps are written nil and skipped on read, extra trailing slots
|
||||
// are read past and noted (decodeNoting): the record that carried them is read-only, never rewritten without them, and a slot missing from the payload takes its declared default or is refused. A default fills only a
|
||||
// are skipped, and a slot missing from the payload takes its declared default or is refused. A default fills only a
|
||||
// missing trailing slot, never a nil: as in the reference, a nil in a reference slot (string, bytes, record, union,
|
||||
// list) reads as null and writes back as nil, so a record the reference wrote re-encodes byte-identical, and a nil in
|
||||
// a value slot (int, float, bool) is refused unless the slot is nullable.
|
||||
|
|
@ -341,9 +281,6 @@ export function slots<TValue extends object>(spec: SlotsSpec<TValue>, typeName?:
|
|||
decode(raw) {
|
||||
expect(raw, "an array of slots", Array.isArray(raw));
|
||||
const input = raw as unknown[];
|
||||
if (input.length > length) {
|
||||
noteExtraSlots(simpleTypeName(typeName), input.length, length);
|
||||
}
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const field of fields) {
|
||||
const present = field.key < input.length;
|
||||
|
|
@ -354,7 +291,7 @@ export function slots<TValue extends object>(spec: SlotsSpec<TValue>, typeName?:
|
|||
? defaultOf(field)
|
||||
: input[field.key] === null && !field.codec.valueType
|
||||
? null
|
||||
: at(`${field.name} (slot ${field.key})`, () => nested(() => field.codec.decode(input[field.key])));
|
||||
: at(`${field.name} (slot ${field.key})`, () => field.codec.decode(input[field.key]));
|
||||
}
|
||||
return out as TValue;
|
||||
},
|
||||
|
|
@ -426,9 +363,6 @@ export function union<TValue extends { kind: string }>(
|
|||
if (arm === undefined) {
|
||||
throw new Error(`Union ${name}: unknown arm key ${key} (declared keys: ${declared}).`);
|
||||
}
|
||||
if (arm.body === unit && raw[1].length > 0) {
|
||||
noteExtraSlots(arm.armName, raw[1].length, 0);
|
||||
}
|
||||
const fields = arm.body === unit ? {} : at(`arm ${arm.armName}`, () => (arm.body as SlotsCodec<any>).decode(raw[1]));
|
||||
return { kind: arm.armName, ...fields } as unknown as TValue;
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { decode } from "@msgpack/msgpack";
|
||||
|
||||
import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth";
|
||||
import { checkPayloadDepth } from "./payload-depth";
|
||||
import { type StoreFormat, isStoreSnapshot, refuseUnreadStoreBytes, requireStoreSlots, requireV1RecordSlots } from "./store-format";
|
||||
|
||||
export interface CultCacheInspection {
|
||||
|
|
@ -235,22 +235,12 @@ function decodeLegacyEnvelope(value: unknown): InspectedRecord {
|
|||
};
|
||||
}
|
||||
|
||||
// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it,
|
||||
// and bytes after the first value are refused with the shared clause (the tail is never decoded or shown).
|
||||
// The inspector's one payload read: the depth walk first, so a payload past the limit is refused before any decoder sees it.
|
||||
function decodeBoundedPayload(payload: Uint8Array): unknown {
|
||||
checkPayloadIsOneValue(payload, "document payload");
|
||||
checkPayloadDepth(payload);
|
||||
return decode(payload);
|
||||
}
|
||||
|
||||
// Where the first value ends (the whole payload when it ends early or cannot be walked): a preview never shows a tail.
|
||||
function valueEnd(payload: Uint8Array): number {
|
||||
try {
|
||||
return checkPayloadDepth(payload) ?? payload.length;
|
||||
} catch {
|
||||
return payload.length;
|
||||
}
|
||||
}
|
||||
|
||||
function previewPayload(payload: Uint8Array): Pick<InspectedRecord, "payloadPreview" | "payloadDecodeError"> {
|
||||
try {
|
||||
return { payloadPreview: toJsonSafe(decodeBoundedPayload(payload)) };
|
||||
|
|
@ -258,7 +248,7 @@ function previewPayload(payload: Uint8Array): Pick<InspectedRecord, "payloadPrev
|
|||
return {
|
||||
payloadPreview: {
|
||||
bytes: payload.length,
|
||||
hexPrefix: [...payload.slice(0, Math.min(32, valueEnd(payload)))]
|
||||
hexPrefix: [...payload.slice(0, 32)]
|
||||
.map((value) => value.toString(16).padStart(2, "0"))
|
||||
.join(" "),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import {
|
||||
declaresRole,
|
||||
decodeDocumentPayloadMarked,
|
||||
decodeDocumentPayload,
|
||||
documentPayloadBytes,
|
||||
documentPayloadFromBytes,
|
||||
encodeDocumentPayload,
|
||||
|
|
@ -8,7 +8,6 @@ import {
|
|||
schemaIdentityOf,
|
||||
} from "./document";
|
||||
import { resolveDocumentFormatter } from "./document-formatter";
|
||||
import type { NotedSlots } from "./codec";
|
||||
import type { DocumentPayload } from "./document";
|
||||
import { SchemaConflictError, schemaIdForEnvelope, writtenOwns } from "./store-format";
|
||||
import type {
|
||||
|
|
@ -21,7 +20,6 @@ import type {
|
|||
CultCacheEnvelope,
|
||||
CultCacheSchemaCatalogEntry,
|
||||
CultCacheStoreRegistration,
|
||||
CultCacheUnreadSlots,
|
||||
} from "./types";
|
||||
|
||||
type Accessor = (value: unknown) => string | undefined;
|
||||
|
|
@ -29,7 +27,7 @@ type Accessor = (value: unknown) => string | undefined;
|
|||
// A registered definition reads and writes its documents through the owner, on payloads the cache has cloned.
|
||||
type RegisteredFormatter = {
|
||||
encode(value: unknown): DocumentPayload;
|
||||
decode(payload: DocumentPayload): { value: unknown; unreadSlots?: NotedSlots };
|
||||
decode(payload: DocumentPayload): unknown;
|
||||
};
|
||||
|
||||
type RegisteredDefinition = {
|
||||
|
|
@ -52,10 +50,6 @@ type HydratedEntry = CultCacheEnvelope & {
|
|||
catalogEntry: NonNullable<CultCacheEnvelope["catalogEntry"]>;
|
||||
value: unknown;
|
||||
lookups: EntryLookups;
|
||||
// What the payload decode read past: set when the record was admitted, and it makes the record read-only unless explained.
|
||||
unreadSlots?: CultCacheUnreadSlots;
|
||||
// The schema id the record arrived under, before it was admitted under its reader's: what an unread-slots refusal names.
|
||||
storedSchemaId?: string;
|
||||
};
|
||||
|
||||
// Everything a pull replaces. Built whole, then swapped in with one assignment.
|
||||
|
|
@ -90,20 +84,6 @@ type DerivedTypeLookups = {
|
|||
readonly indexMaps: Map<string, Map<string, string>>;
|
||||
};
|
||||
|
||||
// The consent a compatible-id declaration gives (cultlib-gaps:ruling:slot-mark-consent): a mark is explained only when the record
|
||||
// arrived under an id other than the reader's own that the reader lists compatible, it is at
|
||||
// depth 0 and the schema comparison of the catalog entry the record arrived with against the reader's own reported the slots it
|
||||
// carries as ignored, so its slot count is at most the highest ignored slot plus one. A record written or put through the cache
|
||||
// arrives under the reader's own entry, which ignores nothing, so no mark of it is explained.
|
||||
function explainedBy(noted: NotedSlots, reader: CultCacheSchemaCatalogEntry, arrivedId: string, arrived: CultCacheSchemaCatalogEntry | undefined): boolean {
|
||||
if (arrivedId === reader.schemaId || !(reader.compatibleSchemaIds ?? []).includes(arrivedId)) {
|
||||
return false;
|
||||
}
|
||||
const declared = new Set((reader.members ?? []).map((member) => member.slot));
|
||||
const ignored = (arrived?.members ?? []).map((member) => member.slot).filter((slot) => !declared.has(slot));
|
||||
return noted.depth0Only && noted.slotCount <= Math.max(-1, ...ignored) + 1;
|
||||
}
|
||||
|
||||
function emptyHydratedState(): HydratedState {
|
||||
return {
|
||||
entries: new Map(),
|
||||
|
|
@ -332,7 +312,7 @@ export class CultCache {
|
|||
}
|
||||
|
||||
const payload = this.#cloneBytes(entry.payload);
|
||||
const { value, unreadSlots } = registered.formatter.decode(payload);
|
||||
const value = registered.formatter.decode(payload);
|
||||
let admitted: CultCacheEnvelope = { ...entry, type, payload };
|
||||
if (registered.global) {
|
||||
const persistedKey = persistedGlobalKeys.get(type);
|
||||
|
|
@ -347,7 +327,7 @@ export class CultCache {
|
|||
admitted = { ...admitted, key: CultCache.GLOBAL_KEY };
|
||||
}
|
||||
}
|
||||
const hydrated = this.#admitEntry(next, registered, admitted, value, unreadSlots);
|
||||
const hydrated = this.#admitEntry(next, registered, admitted, value);
|
||||
const other = this.#heldUnderOtherType(next, type, hydrated.key);
|
||||
if (other !== undefined) {
|
||||
throw new Error(`Record ${hydrated.key} is held by two stores, as type "${other.type}" and as type "${type}".`);
|
||||
|
|
@ -538,9 +518,10 @@ export class CultCache {
|
|||
if (!envelope.storedAt || envelope.storedAt.trim().length === 0) {
|
||||
throw new Error(`CultCache envelope storedAt for type "${definition.type}" must be non-empty.`);
|
||||
}
|
||||
this.#requireGlobalKey(registered, envelope.key);
|
||||
|
||||
const payload = this.#cloneBytes(envelope.payload);
|
||||
const { value: parsed, unreadSlots } = registered.formatter.decode(payload) as { value: CultCacheDocumentValue<TDefinition>; unreadSlots?: NotedSlots };
|
||||
const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue<TDefinition>;
|
||||
const entry: CultCacheEnvelope = {
|
||||
key: envelope.key,
|
||||
type: envelope.type,
|
||||
|
|
@ -551,7 +532,7 @@ export class CultCache {
|
|||
schemaId: registered.catalogEntry.schemaId,
|
||||
catalogEntry: registered.catalogEntry,
|
||||
};
|
||||
await this.#writeNow(registered, entry, parsed, unreadSlots);
|
||||
await this.#writeNow(registered, entry, parsed);
|
||||
return parsed;
|
||||
});
|
||||
}
|
||||
|
|
@ -604,29 +585,6 @@ export class CultCache {
|
|||
});
|
||||
}
|
||||
|
||||
/** What the record's payload carried past this build's declared slots, or undefined: the held record is read-only unless `explained`. */
|
||||
unreadSlots(definition: AnyCultCacheDocumentDefinition, key: string): CultCacheUnreadSlots | undefined {
|
||||
return this.#hydrated.entries.get(this.#entryId(this.#requireDefinition(definition).definition.type, key))?.unreadSlots;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a write of the record's own type at the key would be let through: it asks the judge #writeNow asks, of the record the
|
||||
* cache holds (a key a store carries foreign, a record of another type the write does not own or cannot replace, a global at
|
||||
* another key, a record carrying unread slots nothing consents to). It answers the WRITE form. A removal asks the judge's
|
||||
* removal form and differs in two states: an owner's `delete` of a record held in another store than its own home succeeds
|
||||
* where this is false (a write would push to the wrong store), and `delete` of a global type at a key other than __global__ is
|
||||
* the plain false of an unheld key where this is false. Live and advisory: the write or removal still decides.
|
||||
*/
|
||||
isWritable(definition: AnyCultCacheDocumentDefinition, key: string): boolean {
|
||||
const registered = this.#requireDefinition(definition);
|
||||
try {
|
||||
this.#judge(registered, key, this.#probe(registered, key));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// What the last pull found that no registered type claims, by key ordinal. A later registration claims a record at the next pull.
|
||||
foreignRecords(): readonly CultCacheForeignRecord[] {
|
||||
return [...this.#hydrated.foreign].sort((left, right) => compareOrdinal(left.key, right.key));
|
||||
|
|
@ -636,13 +594,6 @@ export class CultCache {
|
|||
return [...this.#hydrated.entries.values()].map((entry) => this.#toEnvelope(entry));
|
||||
}
|
||||
|
||||
// The image as the stores hold it, for persisting it whole (a flush): a global loaded from a legacy key is handed out under
|
||||
// that key until its first write retires it, so a flush cannot leave the store holding it under two keys. `snapshot` is the
|
||||
// logical image and always names a global by CultCache.GLOBAL_KEY.
|
||||
storedSnapshot(): CultCacheEnvelope[] {
|
||||
return [...this.#hydrated.entries.values()].map((entry) => this.#hydrated.legacyGlobals.get(entry.type) ?? this.#toEnvelope(entry));
|
||||
}
|
||||
|
||||
#registerNow(definition: AnyCultCacheDocumentDefinition): void {
|
||||
// Every registration refusal is a SchemaConflictError naming the new definition's schema id and both schema names.
|
||||
const refuse = (holder: RegisteredDefinition, claimed: CultCacheSchemaCatalogEntry, what: string) =>
|
||||
|
|
@ -729,8 +680,9 @@ export class CultCache {
|
|||
value: CultCacheDocumentValue<TDefinition>,
|
||||
): Promise<CultCacheDocumentValue<TDefinition>> {
|
||||
const registered = this.#requireDefinition(definition);
|
||||
this.#requireGlobalKey(registered, key);
|
||||
const payload = registered.formatter.encode(value);
|
||||
const { value: parsed, unreadSlots } = registered.formatter.decode(payload) as { value: CultCacheDocumentValue<TDefinition>; unreadSlots?: NotedSlots };
|
||||
const parsed = registered.formatter.decode(payload) as CultCacheDocumentValue<TDefinition>;
|
||||
const entry: CultCacheEnvelope = {
|
||||
key,
|
||||
type: definition.type,
|
||||
|
|
@ -739,16 +691,26 @@ export class CultCache {
|
|||
schemaId: registered.catalogEntry.schemaId,
|
||||
catalogEntry: registered.catalogEntry,
|
||||
};
|
||||
await this.#writeNow(registered, entry, parsed, unreadSlots);
|
||||
await this.#writeNow(registered, entry, parsed);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
// Validates the whole write (home, global singleton, name and index accessors) before the store
|
||||
// is touched; once the store accepts, applying to the cache has no fallible step.
|
||||
async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown, unreadSlots?: NotedSlots): Promise<void> {
|
||||
async #writeNow(registered: RegisteredDefinition, entry: CultCacheEnvelope, value: unknown): Promise<void> {
|
||||
const home = this.#homeStore(entry.type);
|
||||
const retyped = this.#judge(registered, entry.key, entry);
|
||||
const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value, unreadSlots);
|
||||
const retyped = this.#refuseRetype(entry);
|
||||
if (retyped !== undefined && this.#homeStore(retyped.type) !== home) {
|
||||
// The write owns the held record's id and would replace it, but the held record is in another store; the push
|
||||
// reaches only this type's home, so the old record would stay in its own file and load as a second holder.
|
||||
throw new SchemaConflictError(
|
||||
schemaIdForEnvelope(entry),
|
||||
[retyped.type, entry.type],
|
||||
entry.key,
|
||||
`CultCache holds record "${entry.key}" as type "${retyped.type}" in another store than the home of type "${entry.type}", so the rename cannot replace it; remove the record first.`,
|
||||
);
|
||||
}
|
||||
const hydrated = this.#admitEntry(this.#hydrated, registered, entry, value);
|
||||
this.#refuseIndexSharing(this.#hydrated, entry.type, registered.catalogEntry.schemaName, entry.key, hydrated.lookups);
|
||||
const legacy = this.#hydrated.legacyGlobals.get(entry.type);
|
||||
await home?.push(entry);
|
||||
|
|
@ -763,85 +725,6 @@ export class CultCache {
|
|||
this.#installEntry(this.#hydrated, registered, hydrated);
|
||||
}
|
||||
|
||||
// An envelope of the definition's type at the key, carrying nothing: what the judge asks about a write or a removal by that type.
|
||||
#probe(registered: RegisteredDefinition, key: string): CultCacheEnvelope {
|
||||
return {
|
||||
key, type: registered.definition.type, payload: documentPayloadFromBytes(new Uint8Array(0)), storedAt: "",
|
||||
schemaId: registered.catalogEntry.schemaId, catalogEntry: registered.catalogEntry,
|
||||
};
|
||||
}
|
||||
|
||||
// The one judge of whether a key may be written (a written envelope) or removed (none), before any store is touched: it throws the
|
||||
// refusal, and #writeNow, #deleteNow and isWritable all ask it, so the query cannot disagree with the write. For a write it
|
||||
// returns the held record of another type that the write owns and so restamps, if any. A refusal names keys and schemas,
|
||||
// never a payload or a slot value.
|
||||
#judge(registered: RegisteredDefinition, key: string, written?: CultCacheEnvelope): HydratedEntry | undefined {
|
||||
const type = registered.definition.type;
|
||||
const held = this.#hydrated.entries.get(this.#entryId(type, key));
|
||||
if (written === undefined) {
|
||||
// A removal is by key: the record it removes is the one held at the key, of this type or of another type whose schema id
|
||||
// this type owns (the record a write of this type would replace). A held record of a type it does not own is refused.
|
||||
this.#refuseForeignKey(key, type, "removed");
|
||||
this.#refuseUnread(held);
|
||||
if (held !== undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const other = this.#heldUnderOtherType(this.#hydrated, type, key);
|
||||
if (other === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (!writtenOwns(this.#probe(registered, key), schemaIdForEnvelope(other))) {
|
||||
throw new SchemaConflictError(
|
||||
registered.catalogEntry.schemaId,
|
||||
[other.type, type],
|
||||
key,
|
||||
`CultCache holds record "${key}" as type "${other.type}", so type "${type}" cannot remove it.`,
|
||||
);
|
||||
}
|
||||
this.#refuseUnread(other);
|
||||
return other;
|
||||
}
|
||||
this.#requireGlobalKey(registered, key);
|
||||
const retyped = this.#refuseRetype(written);
|
||||
this.#refuseUnread(held);
|
||||
this.#refuseUnread(retyped);
|
||||
if (retyped !== undefined && this.#homeStore(retyped.type) !== this.#homeStore(type)) {
|
||||
// The write owns the held record's id and would replace it, but the held record is in another store; the push
|
||||
// reaches only this type's home, so the old record would stay in its own file and load as a second holder.
|
||||
throw new SchemaConflictError(
|
||||
schemaIdForEnvelope(written),
|
||||
[retyped.type, type],
|
||||
key,
|
||||
`CultCache holds record "${key}" as type "${retyped.type}" in another store than the home of type "${type}", so the rename cannot replace it; remove the record first.`,
|
||||
);
|
||||
}
|
||||
return retyped;
|
||||
}
|
||||
|
||||
// The one refusal of a record held read-only for the slots its payload carries past this build's: a write or a removal of
|
||||
// its key. It names the type and the counts, never a slot value. A mark that is explained (see #admitEntry) is consented to.
|
||||
#unreadRefusal(held: HydratedEntry | undefined): SchemaConflictError | undefined {
|
||||
const unread = held?.unreadSlots;
|
||||
if (held === undefined || unread === undefined || unread.explained) {
|
||||
return undefined;
|
||||
}
|
||||
return new SchemaConflictError(
|
||||
held.storedSchemaId ?? held.schemaId ?? held.type,
|
||||
[held.catalogEntry.schemaName],
|
||||
held.key,
|
||||
`${held.catalogEntry.schemaName} record ${held.key} is read-only: ${unread.typeName} was read with ${unread.slotCount} slots where this build declares ${unread.declaredCount} ` +
|
||||
`(${unread.occurrences} object(s) in the record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or write it from that build.`,
|
||||
unread,
|
||||
);
|
||||
}
|
||||
|
||||
#refuseUnread(held: HydratedEntry | undefined): void {
|
||||
const refusal = this.#unreadRefusal(held);
|
||||
if (refusal !== undefined) {
|
||||
throw refusal;
|
||||
}
|
||||
}
|
||||
|
||||
// The one key-identity judge of the cache: may this write land at its key, given what the cache holds there under
|
||||
// every registered type and the keys its stores carry foreign. The store judges the file with the same predicate
|
||||
// (writtenOwns); this judges the cache, which a store-less cache has no other judge for. Returns the held record of
|
||||
|
|
@ -899,20 +782,18 @@ export class CultCache {
|
|||
key: string,
|
||||
): Promise<boolean> {
|
||||
const registered = this.#requireDefinition(definition);
|
||||
// variants:ruling:unheld-removal-refused-at-commit: a key a store carries foreign is refused, a key held by nobody is the
|
||||
// existing false; a key held as another type is removed by an owner of its id and refused to any other type.
|
||||
// Removal is by key, in one store call: the record held at the key of this type, else the one of another type this type owns.
|
||||
const owned = this.#judge(registered, key);
|
||||
const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key)) ?? owned;
|
||||
const entry = this.#hydrated.entries.get(this.#entryId(definition.type, key));
|
||||
if (!entry) {
|
||||
// variants:ruling:unheld-removal-refused-at-commit: a key a store carries foreign is refused, a key held under
|
||||
// another type or by nobody is the existing false.
|
||||
this.#refuseForeignKey(key, definition.type, "removed");
|
||||
return false;
|
||||
}
|
||||
|
||||
// The cache drops the record only once the store has removed it.
|
||||
const home = this.#homeStore(entry.type);
|
||||
await home?.delete(this.#hydrated.legacyGlobals.get(entry.type) ?? this.#toEnvelope(entry));
|
||||
this.#hydrated.legacyGlobals.delete(entry.type);
|
||||
this.#removeEntry(this.#hydrated, this.#definitions.get(entry.type)!, entry);
|
||||
const home = this.#homeStore(definition.type);
|
||||
await home?.delete(this.#hydrated.legacyGlobals.get(definition.type) ?? this.#toEnvelope(entry));
|
||||
this.#hydrated.legacyGlobals.delete(definition.type);
|
||||
this.#removeEntry(this.#hydrated, registered, entry);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -939,7 +820,7 @@ export class CultCache {
|
|||
|
||||
return {
|
||||
encode: (value: unknown) => this.#cloneBytes(encodeDocumentPayload(definition, value)),
|
||||
decode: (payload: DocumentPayload) => decodeDocumentPayloadMarked(definition, this.#cloneBytes(payload)),
|
||||
decode: (payload: DocumentPayload) => decodeDocumentPayload(definition, this.#cloneBytes(payload)),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1119,7 +1000,6 @@ export class CultCache {
|
|||
registered: RegisteredDefinition,
|
||||
entry: CultCacheEnvelope,
|
||||
value: unknown,
|
||||
noted?: NotedSlots,
|
||||
): HydratedEntry {
|
||||
if (registered.global) {
|
||||
const currentGlobalKey = state.globalKeys.get(entry.type);
|
||||
|
|
@ -1140,8 +1020,6 @@ export class CultCache {
|
|||
catalogEntry: registered.catalogEntry,
|
||||
value,
|
||||
lookups,
|
||||
storedSchemaId: entry.schemaId ?? entry.type,
|
||||
...(noted ? { unreadSlots: { ...noted, explained: explainedBy(noted, registered.catalogEntry, entry.schemaId ?? entry.type, entry.catalogEntry) } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { decode, encode } from "@msgpack/msgpack";
|
||||
|
||||
import { MAX_PAYLOAD_DEPTH } from "./payload-depth";
|
||||
import { decodeNoting, type NotedSlots, type SlotsCodec } from "./codec";
|
||||
import type { SlotsCodec } from "./codec";
|
||||
import type { AnyCultCacheDocumentDefinition, CultCacheDocumentFormatter } from "./types";
|
||||
import { readWire, writeWire } from "./wire";
|
||||
|
||||
|
|
@ -16,13 +16,6 @@ export function resolveDocumentFormatter(definition: AnyCultCacheDocumentDefinit
|
|||
return definition.formatter ?? (definition.codec ? codecFormatter(definition.codec) : plainDocumentFormatter);
|
||||
}
|
||||
|
||||
// The one decode of a document's bytes, with the slots it read past. A plain formatter or a user formatter has no slot codec
|
||||
// and carries everything, so it notes nothing.
|
||||
export function decodeMarked(definition: AnyCultCacheDocumentDefinition, bytes: Uint8Array): { value: unknown; unreadSlots?: NotedSlots } {
|
||||
const formatter = resolveDocumentFormatter(definition);
|
||||
return decodeNoting(() => formatter.decode(bytes));
|
||||
}
|
||||
|
||||
const plainDocumentFormatter: CultCacheDocumentFormatter<unknown> = {
|
||||
// The encoder's own depth refusal (100 by default) is set above the limit, so the payload walk decides.
|
||||
encode: (value: unknown) => encode(value, { maxDepth: MAX_PAYLOAD_DEPTH * 2 }),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import type { NotedSlots } from "./codec";
|
||||
import { decodeMarked, resolveDocumentFormatter } from "./document-formatter";
|
||||
import { checkPayloadDepth, checkPayloadIsOneValue } from "./payload-depth";
|
||||
import { resolveDocumentFormatter } from "./document-formatter";
|
||||
import { checkPayloadDepth } from "./payload-depth";
|
||||
import type {
|
||||
AnyCultCacheDocumentDefinition,
|
||||
CultCacheSchemaCatalogMember,
|
||||
|
|
@ -119,24 +118,10 @@ export function decodeDocumentPayload<TDefinition extends AnyCultCacheDocumentDe
|
|||
definition: TDefinition,
|
||||
payload: DocumentPayload,
|
||||
): CultCacheDocumentValue<TDefinition> {
|
||||
return decodeDocumentPayloadMarked(definition, payload).value;
|
||||
}
|
||||
|
||||
// The same read with the slots it read past (cult-cache-ts slot-readonly). The depth check runs first: a payload past the
|
||||
// limit is refused, never marked.
|
||||
export function decodeDocumentPayloadMarked<TDefinition extends AnyCultCacheDocumentDefinition>(
|
||||
definition: TDefinition,
|
||||
payload: DocumentPayload,
|
||||
): { value: CultCacheDocumentValue<TDefinition>; unreadSlots?: NotedSlots } {
|
||||
const bytes = documentPayloadBytes(payload);
|
||||
if (writesLibraryMessagePack(definition)) {
|
||||
// A payload is exactly one MessagePack value: bytes after the first are refused by count before any formatter reads,
|
||||
// so a tail is refused before any slot is marked.
|
||||
checkPayloadIsOneValue(bytes, `document type "${definition.type}" payload`);
|
||||
}
|
||||
const { value: decoded, unreadSlots } = decodeMarked(definition, bytes);
|
||||
const value = definition.schema.parse(decoded) as CultCacheDocumentValue<TDefinition>;
|
||||
return unreadSlots ? { value, unreadSlots } : { value };
|
||||
if (writesLibraryMessagePack(definition)) checkPayloadDepth(bytes, `document type "${definition.type}" payload`);
|
||||
const decoded = resolveDocumentFormatter(definition).decode(bytes);
|
||||
return definition.schema.parse(decoded) as CultCacheDocumentValue<TDefinition>;
|
||||
}
|
||||
|
||||
// The one writer: the definition's schema first, then its formatter, codec or plain map.
|
||||
|
|
|
|||
|
|
@ -12,9 +12,8 @@ export class CultCachePayloadDepthError extends Error {
|
|||
|
||||
// Walks the headers of one MessagePack value iteratively (a count stack bounded by the limit, scalars skipped by length,
|
||||
// nothing read) and throws at the first container past the limit. The message names the subject, the depth and the limit,
|
||||
// never payload bytes. Returns the offset where the first value ends, or undefined when the payload ends early (the
|
||||
// reader's to refuse); what follows that offset is never walked, and whether it may exist is the caller's rule.
|
||||
export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): number | undefined {
|
||||
// never payload bytes. A payload that ends early is the reader's to refuse.
|
||||
export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload"): void {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const remaining = new Float64Array(MAX_PAYLOAD_DEPTH);
|
||||
let depth = 0;
|
||||
|
|
@ -34,12 +33,12 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload
|
|||
else if (code <= 0x8f) { container = true; header = 1; children = 2 * (code & 0x0f); }
|
||||
else if (code <= 0x9f) { container = true; header = 1; children = code & 0x0f; }
|
||||
else if (code <= 0xbf) skip = 1 + (code & 0x1f);
|
||||
else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return undefined; skip = 2 + n; }
|
||||
else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return undefined; skip = 3 + n; }
|
||||
else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return undefined; skip = 5 + n; }
|
||||
else if (code === 0xc7) { const n = width(1); if (n === undefined) return undefined; skip = 3 + n; }
|
||||
else if (code === 0xc8) { const n = width(2); if (n === undefined) return undefined; skip = 4 + n; }
|
||||
else if (code === 0xc9) { const n = width(4); if (n === undefined) return undefined; skip = 6 + n; }
|
||||
else if (code === 0xc4 || code === 0xd9) { const n = width(1); if (n === undefined) return; skip = 2 + n; }
|
||||
else if (code === 0xc5 || code === 0xda) { const n = width(2); if (n === undefined) return; skip = 3 + n; }
|
||||
else if (code === 0xc6 || code === 0xdb) { const n = width(4); if (n === undefined) return; skip = 5 + n; }
|
||||
else if (code === 0xc7) { const n = width(1); if (n === undefined) return; skip = 3 + n; }
|
||||
else if (code === 0xc8) { const n = width(2); if (n === undefined) return; skip = 4 + n; }
|
||||
else if (code === 0xc9) { const n = width(4); if (n === undefined) return; skip = 6 + n; }
|
||||
else if (code === 0xca || code === 0xce || code === 0xd2) skip = 5;
|
||||
else if (code === 0xcb || code === 0xcf || code === 0xd3) skip = 9;
|
||||
else if (code === 0xcc || code === 0xd0) skip = 2;
|
||||
|
|
@ -48,10 +47,10 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload
|
|||
else if (code === 0xd6) skip = 6;
|
||||
else if (code === 0xd7) skip = 10;
|
||||
else if (code === 0xd8) skip = 18;
|
||||
else if (code === 0xdc) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = n; }
|
||||
else if (code === 0xdd) { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = n; }
|
||||
else if (code === 0xde) { const n = width(2); if (n === undefined) return undefined; container = true; header = 3; children = 2 * n; }
|
||||
else { const n = width(4); if (n === undefined) return undefined; container = true; header = 5; children = 2 * n; }
|
||||
else if (code === 0xdc) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = n; }
|
||||
else if (code === 0xdd) { const n = width(4); if (n === undefined) return; container = true; header = 5; children = n; }
|
||||
else if (code === 0xde) { const n = width(2); if (n === undefined) return; container = true; header = 3; children = 2 * n; }
|
||||
else { const n = width(4); if (n === undefined) return; container = true; header = 5; children = 2 * n; }
|
||||
|
||||
if (container) {
|
||||
if (depth === MAX_PAYLOAD_DEPTH) throw new CultCachePayloadDepthError(MAX_PAYLOAD_DEPTH + 1, MAX_PAYLOAD_DEPTH, subject);
|
||||
|
|
@ -64,20 +63,8 @@ export function checkPayloadDepth(bytes: Uint8Array, subject = "document payload
|
|||
position += skip;
|
||||
}
|
||||
|
||||
// A scalar whose declared length runs past the payload is an early end, not a value.
|
||||
if (position > bytes.length) return undefined;
|
||||
// One element is complete; close every container it was the last element of.
|
||||
while (depth > 0 && --remaining[depth - 1] === 0) depth--;
|
||||
if (depth === 0) return position;
|
||||
if (depth === 0) return;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// The one rule that a payload is exactly one MessagePack value: the depth walk, then a refusal of bytes after the first
|
||||
// value, by count, naming the subject, from the walk's end. The tail is never walked or echoed; a payload that ends early
|
||||
// stays the decoder's to refuse. decodeDocumentPayload and the inspector speak this clause and no other.
|
||||
export function checkPayloadIsOneValue(bytes: Uint8Array, subject: string): number | undefined {
|
||||
const end = checkPayloadDepth(bytes, subject);
|
||||
if (end !== undefined && end < bytes.length) throw new Error(`${subject} has ${bytes.length - end} bytes after its first MessagePack value.`);
|
||||
return end;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,24 +82,31 @@ export class SingleFileMessagePackBackingStore implements CacheBackingStore {
|
|||
|
||||
async pushAll(entries: CultCacheEnvelope[], options: PushAllOptions = {}): Promise<void> {
|
||||
await this.#enqueue(async () => {
|
||||
// A flush writes `entries` onto the store as it is under the lock, the way a push does: a record under the same key
|
||||
// is replaced and no other is removed, so a cache whose image is stale (or empty, never pulled) cannot drop what
|
||||
// another writer added. The store is read by the same reader as `pullAll`, so one it would refuse (not exactly one
|
||||
// store, a variant, a body it cannot decode) is refused and left as it is. A soft flush asks that reader too: nothing
|
||||
// at the path is written, a store is left alone.
|
||||
// The reader owns the verdict on what is at the path: a dangling link is refused here as it is for open and push.
|
||||
const disk = this.#readDisk();
|
||||
if (options.soft && !disk.absent) {
|
||||
return;
|
||||
// A flush of the whole store writes the header the file on disk carries, read now: a file marked for element ids
|
||||
// stays marked, and one that is not (or is gone or legacy) is written unmarked. The file is read by the same
|
||||
// reader as `pullAll`, so one it would refuse (not exactly one store, a variant, a body it cannot decode) is refused
|
||||
// and left as it is. A soft flush asks that reader too: nothing at the path is written, a store is left alone.
|
||||
try {
|
||||
const disk = this.#readDisk();
|
||||
if (options.soft && !disk.absent) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.#format = disk.format;
|
||||
} catch (error) {
|
||||
// Only a dangling link reaches here with ENOENT: it holds no store, so a whole-store flush has no header to keep and
|
||||
// replaces the link (R3 pins this). A soft flush does not replace what it cannot read, and every merging writer
|
||||
// has already refused the link when it read the current entries.
|
||||
if (options.soft || (error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.#format = STORE_FORMAT_VERSION;
|
||||
}
|
||||
|
||||
this.#format = disk.format;
|
||||
const held = disk.envelopes;
|
||||
|
||||
requireOneTypePerKey(this.filePath, entries, []);
|
||||
// A key is held once: of same-type entries sharing a key in one batch, the last is kept.
|
||||
const supplied = [...new Map(entries.map((entry) => [entry.key, entry])).values()];
|
||||
requireOneTypePerKey(this.filePath, entries, held);
|
||||
await this.#writeAll(supplied, held);
|
||||
await this.#writeAll([...new Map(entries.map((entry) => [entry.key, entry])).values()], []);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { decode } from "@msgpack/msgpack";
|
||||
|
||||
import type { CultCacheEnvelope, CultCacheUnreadSlots } from "./types";
|
||||
import type { CultCacheEnvelope } from "./types";
|
||||
|
||||
export const STORE_FORMAT_VERSION = "cultcache.store.v1";
|
||||
|
||||
|
|
@ -83,16 +83,13 @@ export class SchemaConflictError extends Error {
|
|||
readonly schemaId: string;
|
||||
readonly schemaNames: string[];
|
||||
readonly recordKey: string;
|
||||
/** Set when the refusal is for unread slots; undefined for every other refusal. */
|
||||
readonly unreadSlots?: CultCacheUnreadSlots;
|
||||
|
||||
constructor(schemaId: string, schemaNames: string[], recordKey: string, message?: string, unreadSlots?: CultCacheUnreadSlots) {
|
||||
constructor(schemaId: string, schemaNames: string[], recordKey: string, message?: string) {
|
||||
super(message ?? `Schema id "${schemaId}" cannot describe record "${recordKey}": ${schemaNames.map((name) => `"${name}"`).join(", ")}.`);
|
||||
this.name = "SchemaConflictError";
|
||||
this.schemaId = schemaId;
|
||||
this.schemaNames = schemaNames;
|
||||
this.recordKey = recordKey;
|
||||
this.unreadSlots = unreadSlots;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -99,26 +99,6 @@ export interface CultCacheEnvelope {
|
|||
catalogEntry?: CultCacheSchemaCatalogEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a payload decode found past the slots this build declares, at any depth: the first type read with extra slots, how many
|
||||
* slots it carried against how many it declares, and how many objects in the record did. It holds no slot value, so a refusal
|
||||
* can name it. A record carrying it is read-only unless `explained`.
|
||||
*/
|
||||
export interface CultCacheUnreadSlots {
|
||||
/** The first type read with extra slots (the unit arm's arm name for a unit arm). */
|
||||
readonly typeName: string;
|
||||
readonly slotCount: number;
|
||||
readonly declaredCount: number;
|
||||
readonly occurrences: number;
|
||||
/** True when every mark sat in the document's own object. */
|
||||
readonly depth0Only: boolean;
|
||||
/**
|
||||
* True only when every mark is at depth 0 and the schema comparison of the id the record was stored under reported every slot
|
||||
* the mark carries (slotCount is at most the highest reported slot plus one): the loss a compatible-id declaration consents to.
|
||||
*/
|
||||
readonly explained: boolean;
|
||||
}
|
||||
|
||||
export interface PushAllOptions {
|
||||
soft?: boolean;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -128,6 +128,7 @@ export function readWire(payload: Uint8Array): unknown {
|
|||
}
|
||||
}
|
||||
|
||||
// One value is read; the caller owns what follows it (decodeDocumentPayload refuses a payload with a tail).
|
||||
return next();
|
||||
const tree = next();
|
||||
if (offset !== payload.length) throw new Error("the MessagePack payload has bytes after its value.");
|
||||
return tree;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,8 @@ const cultcacheRsRoot = existsSync(resolve(cultLibRoot, "packages", "cultcache-r
|
|||
? resolve(cultLibRoot, "packages", "cultcache-rs")
|
||||
: resolve(cultCacheTsRoot, "..", "cultcache-rs");
|
||||
const rustInteropBinary = resolve(
|
||||
process.env.CARGO_TARGET_DIR ?? resolve(cultcacheRsRoot, "target"),
|
||||
cultcacheRsRoot,
|
||||
"target",
|
||||
"debug",
|
||||
"examples",
|
||||
process.platform === "win32" ? "cultcache_interop.exe" : "cultcache_interop",
|
||||
|
|
@ -1018,7 +1019,6 @@ test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async ()
|
|||
assert.deepEqual(reloaded.getRequiredGlobal(settingsDocument), { t: "new" });
|
||||
|
||||
// Deleting an adopted global removes the legacy record.
|
||||
await rm(storePath);
|
||||
await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy]);
|
||||
const deleting = build();
|
||||
await deleting.pullAllBackingStores();
|
||||
|
|
@ -1026,30 +1026,10 @@ test("legacyKeyGlobalLoadsWithoutWritingAndFirstWriteLeavesOneGlobal", async ()
|
|||
assert.deepEqual(await new SingleFileMessagePackBackingStore(storePath).pullAll(), []);
|
||||
|
||||
// Two globals of one type on disk, under any keys, are refused.
|
||||
await rm(storePath);
|
||||
await new SingleFileMessagePackBackingStore(storePath).pushAll([legacy, { ...legacy, key: CultCache.GLOBAL_KEY }]);
|
||||
await assert.rejects(build().pullAllBackingStores(), /has multiple persisted entries/u);
|
||||
});
|
||||
|
||||
test("a flush of a cache that loaded a legacy-key global leaves the store reopenable", async () => {
|
||||
const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ t: z.string() }), global: true });
|
||||
const storePath = join(await mkdtemp(join(tmpdir(), "cultcache-legacy-flush-")), "generic.cc");
|
||||
const store = new SingleFileMessagePackBackingStore(storePath);
|
||||
const build = () => CultCache.builder().withRegistry(defineDocumentRegistry(settingsDocument)).withGenericStore(store).build();
|
||||
const seed = CultCache.builder().withRegistry(defineDocumentRegistry(settingsDocument)).build();
|
||||
await seed.putGlobal(settingsDocument, { t: "old" });
|
||||
await store.push({ ...seed.getRequiredEnvelope(settingsDocument, CultCache.GLOBAL_KEY), key: "legacy" });
|
||||
|
||||
const cache = build();
|
||||
await cache.pullAllBackingStores();
|
||||
assert.equal(cache.snapshot()[0]!.key, CultCache.GLOBAL_KEY, "the logical image names the global by the global key");
|
||||
await store.pushAll(cache.storedSnapshot());
|
||||
assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["legacy"], "the flush wrote the record under the key the store holds it");
|
||||
const reopened = build();
|
||||
await reopened.pullAllBackingStores();
|
||||
assert.deepEqual(reopened.getRequiredGlobal(settingsDocument), { t: "old" });
|
||||
});
|
||||
|
||||
test("concurrentPutAndAttachCannotLandRecordInNonHomeStore", async () => {
|
||||
const settingsDocument = defineDocumentType({ type: "settings", schema: z.object({ t: z.string() }) });
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "cultcache-put-attach-race-"));
|
||||
|
|
@ -1272,48 +1252,6 @@ test("CultCache v1 MessagePack stores are readable across TS, Rust, C#, and Pyth
|
|||
}
|
||||
});
|
||||
|
||||
// A Bifrost-shaped store (a union field, nested bytes) written by one of C# and Rust is opened by the other: C# reads through
|
||||
// its store, Rust through the read-only snapshot door, which takes no lock and writes nothing. Leftover staging files from an
|
||||
// interrupted write of either runtime sit beside the store; a read neither deletes them nor lets them change what it reads, and
|
||||
// the store file's bytes are the same after every read.
|
||||
test("A store written by C# or Rust with a union and nested bytes is read back equal by the other, read-only, past leftover staging files", async () => {
|
||||
await buildInteropPeers();
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "cultcache-shapes-"));
|
||||
const rustRead = (file: string) => runJsonCommand("rust-read-shapes", rustInteropBinary, ["read-shapes", "--file", file], cultcacheRsRoot);
|
||||
const csharpRead = (file: string) => runJsonCommand("csharp-read-shapes", dotnetCommand, [csharpInteropDll, "read-shapes", "--file", file], cultLibRoot);
|
||||
const peers = [
|
||||
{
|
||||
writer: "csharp",
|
||||
write: (file: string) => runJsonCommand("csharp-write-shapes", dotnetCommand, [csharpInteropDll, "write-shapes", "--file", file], cultLibRoot),
|
||||
read: rustRead,
|
||||
},
|
||||
{
|
||||
writer: "rust",
|
||||
write: (file: string) => runJsonCommand("rust-write-shapes", rustInteropBinary, ["write-shapes", "--file", file], cultcacheRsRoot),
|
||||
read: csharpRead,
|
||||
},
|
||||
];
|
||||
for (const { writer, write, read } of peers) {
|
||||
const file = join(tempDir, `${writer}.cc`);
|
||||
const written = await write(file);
|
||||
assert.equal(written.length, 2);
|
||||
// The leftovers an interrupted write leaves: C# stages as `.<name>.<guid N>.tmp`, Rust as `<name>.<guid>.tmp`; both hold partial bytes.
|
||||
const leftovers = [
|
||||
join(tempDir, `.${writer}.cc.0123456789abcdef0123456789abcdef.tmp`),
|
||||
join(tempDir, `${writer}.cc.00000000-0000-4000-8000-000000000000.tmp`),
|
||||
];
|
||||
for (const leftover of leftovers) {
|
||||
await writeFile(leftover, Buffer.from([0x93, 0xb2]));
|
||||
}
|
||||
const before = await readFile(file);
|
||||
assert.deepEqual(await read(file), written, `${writer}'s store did not read back equal`);
|
||||
assert.deepEqual(await readFile(file), before, `reading ${writer}'s store changed its bytes`);
|
||||
for (const leftover of leftovers) {
|
||||
assert.ok(await exists(leftover), `reading ${writer}'s store removed ${leftover}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Records and catalog entries a runtime does not own survive its writes. C# writes a store and each other runtime writes its own
|
||||
// note into it: C# reads its note back, and its record and catalog entry are as C# wrote them. C# then writes again, and the other
|
||||
// runtime's record and catalog entry are as that runtime wrote them. Identity is compared on the decoded MessagePack values, every
|
||||
|
|
@ -2323,7 +2261,7 @@ test("a stamp this runtime reads, followed by bytes that are not UTF-8, gets the
|
|||
}
|
||||
});
|
||||
|
||||
test("pushAll leaves a store it reads alone when soft, and writes onto one it reads otherwise", async () => {
|
||||
test("pushAll leaves a store it reads alone when soft, and replaces one it reads otherwise", async () => {
|
||||
const vector = await readFile(resolve(cultLibRoot, "tests", "vectors", "document-variants-c2a", "v3-base.msgpack"));
|
||||
const envelope = { key: "probe", type: "probe", payload: documentPayloadFromBytes(new Uint8Array([0x80])), storedAt: "2026-10-22T00:00:00Z" };
|
||||
const existing = join(await mkdtemp(join(tmpdir(), "cultcache-pushall-soft-")), "store.msgpack");
|
||||
|
|
@ -2335,7 +2273,7 @@ test("pushAll leaves a store it reads alone when soft, and writes onto one it re
|
|||
const store = new SingleFileMessagePackBackingStore(own);
|
||||
await store.pushAll([envelope]);
|
||||
await store.pushAll([{ ...envelope, key: "second" }]);
|
||||
assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["probe", "second"]);
|
||||
assert.deepEqual((await store.pullAll()).map((entry) => entry.key), ["second"]);
|
||||
});
|
||||
|
||||
test("SingleFileMessagePackBackingStore reads only a missing store as empty", async () => {
|
||||
|
|
@ -2525,15 +2463,14 @@ test("R3 decides resolve-or-refuse: a push through a live link replaces the link
|
|||
assert.deepEqual(await readFile(target), before);
|
||||
});
|
||||
|
||||
test("a pushAll through a dangling link is refused like push, and the link and its volume are left as they were", { skip: process.platform === "win32" }, async () => {
|
||||
test("R3 decides resolve-or-refuse: a pushAll through a dangling link replaces the link", { skip: process.platform === "win32" }, async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "cultcache-dangling-push-all-"));
|
||||
const volume = join(dir, "volume");
|
||||
await mkdir(volume);
|
||||
const link = join(dir, "store.cc");
|
||||
await symlink(join(volume, "store.cc"), link);
|
||||
const envelope = { key: "a", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" };
|
||||
await assert.rejects(() => new SingleFileMessagePackBackingStore(link).pushAll([envelope]), (error: NodeJS.ErrnoException) => error.code === "ENOENT");
|
||||
assert.ok((await lstat(link)).isSymbolicLink());
|
||||
await new SingleFileMessagePackBackingStore(link).pushAll([{ key: "a", type: "t", payload: documentPayloadFromBytes(Uint8Array.of(0x90)), storedAt: "2026-09-30T00:00:00Z" }]);
|
||||
assert.ok((await lstat(link)).isFile());
|
||||
assert.deepEqual(await readdir(volume), []);
|
||||
});
|
||||
|
||||
|
|
@ -2638,7 +2575,7 @@ test("SingleFileMessagePackBackingStore pushAll refuses a store whose header it
|
|||
|
||||
// A file is replaced by a rewrite exactly when this runtime's own reader opens it: one verdict per file, asked by pullAll,
|
||||
// pushAll, soft pushAll, push and delete alike; a soft flush leaves the files that read as they are. The bytes and every runtime's verdict: tests/vectors/document-variants-c2a/readability.
|
||||
test("SingleFileMessagePackBackingStore rewrites a file exactly when it reads", async () => {
|
||||
test("SingleFileMessagePackBackingStore replaces a file exactly when it reads", async () => {
|
||||
const root = join(c2aVectors, "readability");
|
||||
const rows = (await readFile(join(root, "manifest.txt"), "utf8"))
|
||||
.split(/\r?\n/u)
|
||||
|
|
@ -2675,7 +2612,6 @@ test("SingleFileMessagePackBackingStore rewrites a file exactly when it reads",
|
|||
? store.push(envelopes[0]!)
|
||||
: store.delete(envelopes[0]!);
|
||||
if (reads) {
|
||||
const heldBefore = operation === "pushAll" ? (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key) : [];
|
||||
await run();
|
||||
if (operation === "pullAll") {
|
||||
continue;
|
||||
|
|
@ -2692,8 +2628,7 @@ test("SingleFileMessagePackBackingStore rewrites a file exactly when it reads",
|
|||
assert.equal((decode(await readFile(file)) as unknown[])[0], vector.includes("v3") ? "cultcache.store.v3" : "cultcache.store.v1", `${vector} ${operation}`);
|
||||
const keys = (await new SingleFileMessagePackBackingStore(file).pullAll()).map((entry) => entry.key);
|
||||
if (operation === "pushAll") {
|
||||
assert.deepEqual(envelopes.map((entry) => entry.key).filter((key) => !keys.includes(key)), [], `${vector} ${operation}: every written key is held`);
|
||||
assert.deepEqual(heldBefore.filter((key) => !keys.includes(key)), [], `${vector} ${operation}: nothing else is removed`);
|
||||
assert.deepEqual(keys, envelopes.map((entry) => entry.key).sort(), `${vector} ${operation}`);
|
||||
} else if (operation === "push") {
|
||||
assert.ok(keys.includes(envelopes[0]!.key), `${vector} ${operation}`);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -119,19 +119,6 @@ test("a payload that ends early, is empty or carries trailing bytes is the reade
|
|||
assert.doesNotThrow(() => checkPayloadDepth(bytes(0xc0)));
|
||||
});
|
||||
|
||||
test("the walk returns where the first value ends, and undefined when the payload ends early", () => {
|
||||
assert.equal(checkPayloadDepth(bytes(0xc0)), 1);
|
||||
assert.equal(checkPayloadDepth(bytes(0xc0, 0xc1, 0xc1)), 1, "a top-level scalar ends at its own length; the tail is not judged");
|
||||
assert.equal(checkPayloadDepth(bytes(0x92, 0x01, 0xa1, 0x61, 0xc0)), 5 - 1, "a container ends when its last child completes");
|
||||
assert.equal(checkPayloadDepth(cat(bytes(0x90), arrays(L + 50))), 1, "the tail is never walked");
|
||||
assert.equal(checkPayloadDepth(bytes()), undefined);
|
||||
assert.equal(checkPayloadDepth(bytes(0x92, 0x01)), undefined);
|
||||
assert.equal(checkPayloadDepth(bytes(0xd9)), undefined);
|
||||
for (const cut of [[0xcc], [0xcb, 0x3f, 0xf8], [0xa5, 0x61, 0x62, 0x63], [0xc4, 0x02, 0x01], [0xd7, 0x01]]) {
|
||||
assert.equal(checkPayloadDepth(bytes(...cut)), undefined, `a scalar cut short: ${cut.map((value) => value.toString(16)).join(" ")}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("the walk is iterative: 100000 nested arrays are refused with the depth error, not a RangeError", () => {
|
||||
refused(arrays(100_000), "100000 nested arrays");
|
||||
refused(chain(bytes(0x81, 0x00), 100_000), "100000 nested maps");
|
||||
|
|
@ -211,13 +198,6 @@ test("the inspector previews a payload at the limit and reports one past it, or
|
|||
}
|
||||
});
|
||||
|
||||
test("the inspector refuses bytes after a payload's first value with the shared clause and previews no tail", async () => {
|
||||
const file = await storeWith(bytes(0xc0, 0xc1, 0xc1));
|
||||
const record = inspectCultCacheBytes(file, await readFile(file)).records[0]!;
|
||||
assert.equal(record.payloadDecodeError, "document payload has 2 bytes after its first MessagePack value.");
|
||||
assert.deepEqual(record.payloadPreview, { bytes: 3, hexPrefix: "c0" });
|
||||
});
|
||||
|
||||
// A header whose length or count needs all four bytes: the low sixteen bits of the length are small, the whole is not.
|
||||
const be32 = (n: number) => bytes((n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,461 +0,0 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { copyFile, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { decode, encode } from "@msgpack/msgpack";
|
||||
import { z } from "zod";
|
||||
|
||||
import { codecs } from "../src";
|
||||
import { CultCache } from "../src/cult-cache";
|
||||
import { defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "../src/document";
|
||||
import { SingleFileMessagePackBackingStore, readSingleFileStore } from "../src/single-file-messagepack-backing-store";
|
||||
import { SchemaConflictError, StoreUnreadableError } from "../src/store-format";
|
||||
import type { CacheBackingStore, CultCacheEnvelope, CultCacheUnreadSlots } from "../src/types";
|
||||
import {
|
||||
type Surface, envelopeOf, golden, hex, memoryStore, read, slotUnitDocument, surfaceCodec, surfaceDeclaringDocument, surfaceDocument, surfaceSchema, vectors,
|
||||
} from "./union-surface-fixture";
|
||||
|
||||
// cultlib-gaps:cut:slot-readonly-ts. A payload carrying a slot past its reader's declared slots, at any depth, is read, held
|
||||
// read-only, and every write or removal of its key is refused before a store is touched. The vectors are written by the C#
|
||||
// reference: tests/vectors/union-parity/README.md, "Extra trailing slot" and "Slot stores".
|
||||
|
||||
const { slots, string } = codecs;
|
||||
const SLOT_VALUE = "extra-slot";
|
||||
|
||||
function mark(typeName: string, slotCount: number, declaredCount: number, depth0Only: boolean, explained = false, occurrences = 1): CultCacheUnreadSlots {
|
||||
return { typeName, slotCount, declaredCount, occurrences, depth0Only, explained };
|
||||
}
|
||||
|
||||
// File, then the mark the README table gives for it.
|
||||
const PAYLOAD_ROWS: [string, CultCacheUnreadSlots][] = [
|
||||
["extra-slot-record.msgpack", mark("UnionSurface", 8, 7, true)],
|
||||
["extra-slot-nested-record.msgpack", mark("Composition", 3, 2, false)],
|
||||
["extra-slot-arm.msgpack", mark("Box", 3, 2, false)],
|
||||
["extra-slot-list-element.msgpack", mark("Text", 3, 2, false)],
|
||||
["depth-skipped-at-limit.msgpack", mark("UnionSurface", 8, 7, true)],
|
||||
];
|
||||
|
||||
async function readOnlyFiles(name: string): Promise<string[]> {
|
||||
const rows = decode(await read(name)) as unknown[][];
|
||||
return rows.filter((row) => row[1] === "read-only").map((row) => row[0] as string);
|
||||
}
|
||||
|
||||
test("the read-only payload vectors are exactly the ones this walker names: four in outcomes, one in bounds", async () => {
|
||||
const listed = [...(await readOnlyFiles("outcomes.msgpack")), ...(await readOnlyFiles("bounds-outcomes.msgpack"))];
|
||||
assert.deepEqual(listed.sort(), PAYLOAD_ROWS.map(([file]) => file).sort());
|
||||
});
|
||||
|
||||
for (const [file, expected] of PAYLOAD_ROWS) {
|
||||
test(`${file}: read past the extra slot, held read-only with the mark, every write and removal of the key refused`, async () => {
|
||||
const document = surfaceDocument();
|
||||
const entries = [envelopeOf(await read(file))];
|
||||
const before = hex(documentPayloadBytes(entries[0]!.payload));
|
||||
const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build();
|
||||
await cache.pullAllBackingStores();
|
||||
|
||||
assert.deepEqual(cache.get(document, "surface"), golden, "the extra slot is read past");
|
||||
assert.deepEqual(cache.unreadSlots(document, "surface"), expected);
|
||||
assert.equal(cache.isWritable(document, "surface"), false);
|
||||
assert.equal(cache.isWritable(document, "another"), true);
|
||||
|
||||
const refused = (error: unknown) => {
|
||||
assert.ok(error instanceof SchemaConflictError, "a SchemaConflictError");
|
||||
assert.deepEqual(error.unreadSlots, expected);
|
||||
assert.equal(error.recordKey, "surface");
|
||||
assert.ok(error.message.includes(expected.typeName), error.message);
|
||||
assert.ok(error.message.includes(`read with ${expected.slotCount} slots where this build declares ${expected.declaredCount} (${expected.occurrences} object(s)`), error.message);
|
||||
assert.ok(!error.message.includes(SLOT_VALUE), "the refusal never holds a slot value");
|
||||
return true;
|
||||
};
|
||||
await assert.rejects(() => cache.put(document, "surface", golden), refused);
|
||||
await assert.rejects(() => cache.update(document, "surface", (value) => ({ ...value!, version: 4 })), refused);
|
||||
const replacement = envelopeOf(await read("union-surface.msgpack"));
|
||||
await assert.rejects(() => cache.putEnvelope(document, replacement), refused);
|
||||
await assert.rejects(() => cache.delete(document, "surface"), refused);
|
||||
assert.equal(entries.length, 1, "nothing reached the store");
|
||||
assert.equal(hex(documentPayloadBytes(entries[0]!.payload)), before, "the held bytes are untouched");
|
||||
assert.deepEqual(cache.get(document, "surface"), golden, "the record is still held");
|
||||
|
||||
await cache.put(document, "another", golden);
|
||||
assert.equal(entries.length, 2, "another key of the same store writes");
|
||||
});
|
||||
}
|
||||
|
||||
test("a clean payload and a short one are writable: missing trailing slots keep the defaults and mark nothing", async () => {
|
||||
const document = surfaceDocument();
|
||||
const short = encode([...(decode(await read("union-surface.msgpack")) as unknown[]).slice(0, 6)]);
|
||||
const entries = [envelopeOf(await read("union-surface.msgpack")), { ...envelopeOf(short), key: "short" }];
|
||||
const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build();
|
||||
await cache.pullAllBackingStores();
|
||||
for (const key of ["surface", "short"]) {
|
||||
assert.equal(cache.unreadSlots(document, key), undefined);
|
||||
assert.equal(cache.isWritable(document, key), true);
|
||||
}
|
||||
await cache.put(document, "short", golden);
|
||||
});
|
||||
|
||||
test("a payload past the nesting limit is refused, never marked", async () => {
|
||||
const entries = [envelopeOf(await read("depth-skipped-past-limit.msgpack"))];
|
||||
const cache = CultCache.builder().withDocumentType(surfaceDocument()).withGenericStore(memoryStore(entries)).build();
|
||||
await assert.rejects(() => cache.pullAllBackingStores(), /nesting|depth/iu);
|
||||
});
|
||||
|
||||
test("putEnvelope of a payload carrying extra slots lands the bytes and holds the record read-only", async () => {
|
||||
const document = surfaceDocument();
|
||||
const entries: ReturnType<typeof envelopeOf>[] = [];
|
||||
const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore(entries)).build();
|
||||
const marked = envelopeOf(await read("extra-slot-nested-record.msgpack"));
|
||||
await cache.putEnvelope(document, marked);
|
||||
assert.equal(hex(documentPayloadBytes(entries[0]!.payload)), hex(await read("extra-slot-nested-record.msgpack")), "the bytes are carried");
|
||||
assert.deepEqual(cache.unreadSlots(document, "surface"), mark("Composition", 3, 2, false));
|
||||
assert.equal(cache.isWritable(document, "surface"), false);
|
||||
await assert.rejects(() => cache.put(document, "surface", golden), SchemaConflictError);
|
||||
});
|
||||
|
||||
test("a key a store carries foreign is not writable", async () => {
|
||||
const document = surfaceDocument();
|
||||
const foreign = { ...envelopeOf(await read("union-surface.msgpack")), key: "stranger", type: "tests.unknown", schemaId: "tests.unknown.v1" };
|
||||
const cache = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([foreign])).build();
|
||||
await cache.pullAllBackingStores();
|
||||
assert.equal(cache.isWritable(document, "stranger"), false);
|
||||
assert.equal(cache.isWritable(document, "other"), true);
|
||||
});
|
||||
|
||||
test("a write by a type that owns the held record's id restamps it, and is refused when that record carries unread slots", async () => {
|
||||
const typed = (type: string, schemaId: string, compatibleSchemaIds: string[] = []) =>
|
||||
defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, compatibleSchemaIds, schema: surfaceSchema, codec: surfaceCodec, name: "id" });
|
||||
const older = typed("tests.surface_older", "tests.surface_older.v1");
|
||||
const newer = typed("tests.surface_newer", "tests.surface_newer.v1", ["tests.surface_older.v1"]);
|
||||
for (const [file, lands] of [["union-surface.msgpack", true], ["extra-slot-record.msgpack", false]] as const) {
|
||||
const held = { ...envelopeOf(await read(file)), type: older.type, schemaId: "tests.surface_older.v1" };
|
||||
const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(memoryStore([held])).build();
|
||||
await cache.pullAllBackingStores();
|
||||
assert.equal(cache.isWritable(newer, "surface"), lands, `${file}: isWritable agrees with the write by the type that owns the id`);
|
||||
const write = cache.put(newer, "surface", golden);
|
||||
if (lands) {
|
||||
await write;
|
||||
} else {
|
||||
await assert.rejects(() => write, (error: unknown) => error instanceof SchemaConflictError && error.unreadSlots?.typeName === "UnionSurface");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// The store vectors. TS has no variant records: extra-slot-store.cc carries one (marked-variant), so the whole file is refused
|
||||
// by name, and the walk runs over the file without that record.
|
||||
async function copyOf(name: string): Promise<string> {
|
||||
const path = join(await mkdtemp(join(tmpdir(), "slot-readonly-")), name);
|
||||
await copyFile(join(vectors, name), path);
|
||||
return path;
|
||||
}
|
||||
|
||||
async function withoutVariant(path: string): Promise<void> {
|
||||
const [format, catalog, records] = decode(await readFile(path)) as [unknown, unknown[], unknown[][]];
|
||||
await writeFile(path, encode([format, catalog, records.filter((record) => record[0] !== "marked-variant")]));
|
||||
}
|
||||
|
||||
type SlotRow = [string, string, boolean, boolean | null, string | null, number | null, number | null, number | null, string | null];
|
||||
|
||||
async function slotRows(): Promise<SlotRow[]> {
|
||||
return decode(await read("slot-outcomes.msgpack")) as SlotRow[];
|
||||
}
|
||||
|
||||
// The catalog entry the file publishes for a schema: its id and the reference's content hash, which a store rewrites with the
|
||||
// definition's own.
|
||||
async function catalogOf(file: string, schemaName: string): Promise<[string, string]> {
|
||||
const [, catalog] = decode(await read(file)) as [unknown, string[][]];
|
||||
const entry = catalog.find((candidate) => candidate[1] === schemaName)!;
|
||||
return [entry[0]!, entry[3]!];
|
||||
}
|
||||
|
||||
async function openStore(file: string, consent: boolean) {
|
||||
const path = await copyOf(file);
|
||||
if (file === "extra-slot-store.cc") await withoutVariant(path);
|
||||
const surface = consent ? surfaceDeclaringDocument() : surfaceDocument(...(await catalogOf(file, "tests.union_surface")));
|
||||
const slotUnit = consent ? undefined : slotUnitDocument(...(await catalogOf(file, "tests.slot_unit")));
|
||||
const builder = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path));
|
||||
const cache = (slotUnit ? builder.withDocumentType(slotUnit) : builder).build();
|
||||
await cache.pullAllBackingStores();
|
||||
return { path, cache, surface, slotUnit };
|
||||
}
|
||||
|
||||
test("extra-slot-store.cc carries a variant, which this runtime refuses by name; the rest of the file is walked", async () => {
|
||||
const path = await copyOf("extra-slot-store.cc");
|
||||
assert.throws(() => readSingleFileStore(path), StoreUnreadableError);
|
||||
});
|
||||
|
||||
test("every slot-outcomes row holds: the mark, IsWritable, and the removal refused exactly where the record is not writable", async () => {
|
||||
const rows = await slotRows();
|
||||
assert.equal(rows.length, 8);
|
||||
let walked = 0;
|
||||
for (const [file, key, writable, explained, typeName, slots, declared, occurrences, flattenRefusedBy] of rows) {
|
||||
if (flattenRefusedBy !== null) continue; // a variant: this runtime holds none
|
||||
walked += 1;
|
||||
const { path, cache, surface, slotUnit } = await openStore(file!, file === "extra-slot-consent.cc");
|
||||
const document = key === "unit-arm" ? slotUnit! : surface;
|
||||
const label = `${file} ${key}`;
|
||||
const found = cache.unreadSlots(document, key!);
|
||||
if (typeName === null) {
|
||||
assert.equal(found, undefined, label);
|
||||
} else {
|
||||
assert.ok(found, label);
|
||||
assert.deepEqual([found.explained, found.typeName, found.slotCount, found.declaredCount, found.occurrences], [explained, typeName, slots, declared, occurrences], label);
|
||||
}
|
||||
assert.equal(cache.isWritable(document, key!), writable, label);
|
||||
|
||||
const original = await readFile(path);
|
||||
if (writable) {
|
||||
assert.equal(await cache.delete(document, key!), true, label);
|
||||
} else {
|
||||
await assert.rejects(
|
||||
() => cache.delete(document, key!),
|
||||
(error: unknown) => error instanceof SchemaConflictError && error.unreadSlots?.typeName === typeName && !error.message.includes(SLOT_VALUE),
|
||||
label,
|
||||
);
|
||||
assert.equal(Buffer.compare(original, await readFile(path)), 0, `${label}: the file is unchanged`);
|
||||
}
|
||||
}
|
||||
assert.equal(walked, 7);
|
||||
});
|
||||
|
||||
test("consent: a write of the explained record lands; the over-consent write is refused; the nested mark is refused", async () => {
|
||||
const consented = await openStore("extra-slot-consent.cc", true);
|
||||
const value = consented.cache.get(consented.surface, "consented") as Surface;
|
||||
assert.deepEqual(value, golden);
|
||||
await consented.cache.put(consented.surface, "consented", value);
|
||||
assert.equal(consented.cache.unreadSlots(consented.surface, "consented"), undefined, "the write dropped the explained slot and the mark with it");
|
||||
assert.equal(readSingleFileStore(consented.path).envelopes.find((e) => e.key === "consented")!.schemaId, "tests.union_surface.v2");
|
||||
|
||||
for (const key of ["over-consent", "nested"]) {
|
||||
const refused = await openStore("extra-slot-consent.cc", true);
|
||||
await assert.rejects(
|
||||
() => refused.cache.put(refused.surface, key, golden),
|
||||
(error: unknown) => error instanceof SchemaConflictError && error.schemaId === "tests.union_surface.wide",
|
||||
`${key}: the refusal names the id the record was stored under`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("carried: after writing another record, the marked record's bytes, schema id, stored-at and every catalog entry are the originals", async () => {
|
||||
const { path, cache, surface } = await openStore("extra-slot-store.cc", false);
|
||||
const originals = readSingleFileStore(path);
|
||||
await cache.put(surface, "clean", golden);
|
||||
const after = readSingleFileStore(path);
|
||||
for (const key of ["marked", "unit-arm"]) {
|
||||
const was = originals.envelopes.find((e) => e.key === key)!;
|
||||
const now = after.envelopes.find((e) => e.key === key)!;
|
||||
assert.equal(hex(documentPayloadBytes(now.payload)), hex(documentPayloadBytes(was.payload)), key);
|
||||
assert.equal(now.schemaId, was.schemaId, key);
|
||||
assert.equal(now.storedAt, was.storedAt, key);
|
||||
assert.deepEqual(now.catalogEntry, was.catalogEntry, key);
|
||||
}
|
||||
});
|
||||
|
||||
// A writer that adds a slot and keeps its schema id (TS ids are hand-written; C# ids hash the members): the reader of that
|
||||
// same id has declared nothing, so it consents to nothing.
|
||||
test("a wider writer under the reader's own schema id is read-only, and a real round trip does not shed the slot", async () => {
|
||||
const doc = (columns: 1 | 2, compatibleSchemaIds: string[] = []) => defineDocumentType({
|
||||
type: "tests.own_id",
|
||||
schemaId: "tests.own_id.v1",
|
||||
schemaName: "tests.own_id",
|
||||
compatibleSchemaIds,
|
||||
schema: z.object({ n: z.string(), extra: z.string().optional() }),
|
||||
codec: columns === 2
|
||||
? slots<{ n: string; extra: string }>({ 0: ["n", string], 1: ["extra", string] }, "Own")
|
||||
: slots<{ n: string }>({ 0: ["n", string] }, "Own"),
|
||||
name: "n",
|
||||
});
|
||||
const path = join(await mkdtemp(join(tmpdir(), "slot-readonly-")), "own.cc");
|
||||
const wide = doc(2);
|
||||
const writer = CultCache.builder().withDocumentType(wide).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||
await writer.put(wide, "k", { n: "n1", extra: "kept" });
|
||||
const bytes = await readFile(path);
|
||||
|
||||
for (const compatible of [[], ["tests.own_id.v1"]]) {
|
||||
const reader = doc(1, compatible);
|
||||
const cache = CultCache.builder().withDocumentType(reader).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||
await cache.pullAllBackingStores();
|
||||
assert.deepEqual(cache.unreadSlots(reader, "k"), mark("Own", 2, 1, true), "unexplained: the arrived id is the reader's own");
|
||||
assert.equal(cache.isWritable(reader, "k"), false);
|
||||
await assert.rejects(() => cache.put(reader, "k", { n: "n2" }), SchemaConflictError);
|
||||
await assert.rejects(() => cache.delete(reader, "k"), SchemaConflictError);
|
||||
assert.equal(Buffer.compare(bytes, await readFile(path)), 0, "the file still holds the slot");
|
||||
}
|
||||
});
|
||||
|
||||
test("a record that arrived under an id the reader does not list compatible is never consented to", async () => {
|
||||
const path = await copyOf("extra-slot-consent.cc");
|
||||
const reader = surfaceDocument("tests.union_surface.v2"); // resolves by type name; lists nothing
|
||||
const cache = CultCache.builder().withDocumentType(reader).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||
await cache.pullAllBackingStores();
|
||||
assert.equal(cache.unreadSlots(reader, "consented")?.explained, false);
|
||||
assert.equal(cache.isWritable(reader, "consented"), false);
|
||||
});
|
||||
|
||||
// isWritable is the write's own judge, not a second one: for every key state it answers what put does.
|
||||
test("isWritable equals the outcome of put for every key state the write judges", async () => {
|
||||
const typed = (type: string, schemaId: string, extra: Record<string, unknown> = {}) =>
|
||||
defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: surfaceSchema, codec: surfaceCodec, name: "id", ...extra });
|
||||
const older = typed("tests.surface_older", "tests.surface_older.v1");
|
||||
const newer = typed("tests.surface_newer", "tests.surface_newer.v1", { compatibleSchemaIds: ["tests.surface_older.v1"] });
|
||||
const stranger = typed("tests.surface_stranger", "tests.surface_stranger.v1");
|
||||
const global = typed("tests.surface_global", "tests.surface_global.v1", { global: true });
|
||||
const clean = (key: string, type: string, schemaId: string) => ({ ...envelopeOf(new Uint8Array()), key, type, schemaId });
|
||||
const held = async (key: string, file: string, type: string, schemaId: string) =>
|
||||
({ ...envelopeOf(await read(file)), key, type, schemaId });
|
||||
|
||||
type Row = { name: string; build: () => Promise<{ cache: CultCache; target: typeof older; key: string }> };
|
||||
const rows: Row[] = [
|
||||
{ name: "owned clean record in the writer's own store", build: async () => {
|
||||
const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer)
|
||||
.withGenericStore(memoryStore([await held("k", "union-surface.msgpack", older.type, older.schemaId!)])).build();
|
||||
return { cache, target: newer, key: "k" };
|
||||
} },
|
||||
{ name: "owned record held in a store other than the writer's home", build: async () => {
|
||||
const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer)
|
||||
.withBackingStore(memoryStore([await held("k", "union-surface.msgpack", older.type, older.schemaId!)]), older)
|
||||
.withBackingStore(memoryStore([]), newer).build();
|
||||
return { cache, target: newer, key: "k" };
|
||||
} },
|
||||
{ name: "a record of a type that does not own the writer's id", build: async () => {
|
||||
const cache = CultCache.builder().withDocumentType(older).withDocumentType(stranger)
|
||||
.withGenericStore(memoryStore([await held("k", "union-surface.msgpack", older.type, older.schemaId!)])).build();
|
||||
return { cache, target: stranger, key: "k" };
|
||||
} },
|
||||
{ name: "a global type at a key other than the global key", build: async () => {
|
||||
const cache = CultCache.builder().withDocumentType(global).withGenericStore(memoryStore([])).build();
|
||||
return { cache, target: global, key: "x" };
|
||||
} },
|
||||
{ name: "a key a store carries foreign", build: async () => {
|
||||
const cache = CultCache.builder().withDocumentType(older)
|
||||
.withGenericStore(memoryStore([{ ...clean("k", "tests.unknown", "tests.unknown.v1"), payload: documentPayloadFromBytes(await read("union-surface.msgpack")) }])).build();
|
||||
return { cache, target: older, key: "k" };
|
||||
} },
|
||||
{ name: "a marked record of the writer's own type", build: async () => {
|
||||
const cache = CultCache.builder().withDocumentType(older)
|
||||
.withGenericStore(memoryStore([await held("k", "extra-slot-record.msgpack", older.type, older.schemaId!)])).build();
|
||||
return { cache, target: older, key: "k" };
|
||||
} },
|
||||
{ name: "an unheld key", build: async () => ({ cache: CultCache.builder().withDocumentType(older).withGenericStore(memoryStore([])).build(), target: older, key: "k" }) },
|
||||
];
|
||||
const outcomes: [string, boolean, boolean][] = [];
|
||||
for (const row of rows) {
|
||||
const { cache, target, key } = await row.build();
|
||||
await cache.pullAllBackingStores();
|
||||
const predicted = cache.isWritable(target, key);
|
||||
const landed = await cache.put(target, key === "x" ? "x" : key, golden).then(() => true, () => false);
|
||||
assert.equal(predicted, landed, row.name);
|
||||
outcomes.push([row.name, predicted, landed]);
|
||||
}
|
||||
assert.deepEqual(outcomes.map(([, writable]) => writable), [true, false, false, false, false, false, true]);
|
||||
// The same judge refuses a removal of a key held read-only or carried foreign.
|
||||
const removals = CultCache.builder().withDocumentType(older)
|
||||
.withGenericStore(memoryStore([await held("k", "extra-slot-record.msgpack", older.type, older.schemaId!)])).build();
|
||||
await removals.pullAllBackingStores();
|
||||
await assert.rejects(() => removals.delete(older, "k"), SchemaConflictError);
|
||||
const stranger2 = CultCache.builder().withDocumentType(older)
|
||||
.withGenericStore(memoryStore([{ ...envelopeOf(await read("union-surface.msgpack")), key: "f", type: "tests.unknown", schemaId: "tests.unknown.v1" }])).build();
|
||||
await stranger2.pullAllBackingStores();
|
||||
await assert.rejects(() => stranger2.delete(older, "f"), SchemaConflictError);
|
||||
});
|
||||
|
||||
// A removal is by key (C# Remove): the owner of a held record's id removes the one record that exists, in one store call and
|
||||
// with the cache dropped only once the store has; a type that does not own it is refused like put, and isWritable agrees.
|
||||
test("delete of a key held as another type: the owner removes it once, a stranger is refused, a failing store leaves cache and store agreeing", async () => {
|
||||
const typed = (type: string, schemaId: string, extra: Record<string, unknown> = {}) =>
|
||||
defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: surfaceSchema, codec: surfaceCodec, name: "id", ...extra });
|
||||
const older = typed("tests.surface_older", "tests.surface_older.v1");
|
||||
const newer = typed("tests.surface_newer", "tests.surface_newer.v1", { compatibleSchemaIds: ["tests.surface_older.v1"] });
|
||||
const stranger = typed("tests.surface_stranger", "tests.surface_stranger.v1");
|
||||
const heldRecord = async (): Promise<CultCacheEnvelope> => ({ ...envelopeOf(await read("union-surface.msgpack")), key: "k", type: older.type, schemaId: older.schemaId });
|
||||
const open = async (failing = false) => {
|
||||
const rows = [await heldRecord()];
|
||||
const calls: string[] = [];
|
||||
const store: CacheBackingStore = {
|
||||
async pullAll() { return rows.map((row) => ({ ...row })); },
|
||||
async push(entry) { rows.push(entry); },
|
||||
async delete(entry) {
|
||||
calls.push(entry.key);
|
||||
if (failing) throw new Error("store refused the removal");
|
||||
rows.splice(rows.findIndex((row) => row.key === entry.key), 1);
|
||||
},
|
||||
};
|
||||
const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withDocumentType(stranger).withGenericStore(store).build();
|
||||
await cache.pullAllBackingStores();
|
||||
return { cache, rows, calls };
|
||||
};
|
||||
|
||||
const owner = await open();
|
||||
assert.equal(owner.cache.isWritable(newer, "k"), true);
|
||||
assert.equal(await owner.cache.delete(newer, "k"), true, "the owner's removal removes the one record");
|
||||
assert.deepEqual([owner.rows.length, owner.calls.length, owner.cache.get(older, "k")], [0, 1, undefined], "one store call, record gone from store and cache");
|
||||
|
||||
const refused = await open();
|
||||
assert.equal(refused.cache.isWritable(stranger, "k"), false);
|
||||
await assert.rejects(() => refused.cache.delete(stranger, "k"), SchemaConflictError);
|
||||
assert.deepEqual([refused.rows.length, refused.calls.length, refused.cache.get(older, "k") !== undefined], [1, 0, true]);
|
||||
|
||||
const failing = await open(true);
|
||||
await assert.rejects(() => failing.cache.delete(newer, "k"), /store refused/u);
|
||||
assert.deepEqual([failing.rows.length, failing.calls.length, failing.cache.get(older, "k") !== undefined], [1, 1, true], "the store failed: neither side dropped it");
|
||||
|
||||
const same = await open();
|
||||
assert.equal(await same.cache.delete(older, "k"), true);
|
||||
assert.equal(await same.cache.delete(older, "k"), false, "an unheld key is still the plain false");
|
||||
});
|
||||
|
||||
// The removal form's three finer behaviours (cultlib-gaps:finding:cut-slot-readonly-ts.s4.removal-form-unpinned).
|
||||
test("an owner's delete: refused for a marked record, taken from the held record's store, and clears the held type's lookups", async () => {
|
||||
const typed = (type: string, schemaId: string, extra: Record<string, unknown> = {}) =>
|
||||
defineDocumentType({ type, schemaId, schemaName: type, schemaVersion: `${type}.v1`, schema: surfaceSchema, codec: surfaceCodec, name: "id", ...extra });
|
||||
const older = typed("tests.surface_older", "tests.surface_older.v1", { indexes: { byVersion: (value: Surface) => String(value.version) } });
|
||||
const newer = typed("tests.surface_newer", "tests.surface_newer.v1", { compatibleSchemaIds: ["tests.surface_older.v1"] });
|
||||
const heldIn = async (file: string): Promise<CultCacheEnvelope> =>
|
||||
({ ...envelopeOf(await read(file)), key: "k", type: older.type, schemaId: older.schemaId });
|
||||
const recording = (rows: CultCacheEnvelope[]) => {
|
||||
const removed: string[] = [];
|
||||
const store: CacheBackingStore = {
|
||||
async pullAll() { return rows.map((row) => ({ ...row })); },
|
||||
async push(entry) { rows.push(entry); },
|
||||
async delete(entry) { removed.push(entry.key); rows.splice(rows.findIndex((row) => row.key === entry.key), 1); },
|
||||
};
|
||||
return { store, removed, rows };
|
||||
};
|
||||
|
||||
// 1. The owner may not remove a record carrying unread slots nothing consents to.
|
||||
const marked = recording([await heldIn("extra-slot-record.msgpack")]);
|
||||
const markedCache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(marked.store).build();
|
||||
await markedCache.pullAllBackingStores();
|
||||
await assert.rejects(() => markedCache.delete(newer, "k"), (error: unknown) => error instanceof SchemaConflictError && error.unreadSlots?.typeName === "UnionSurface");
|
||||
assert.deepEqual([marked.rows.length, marked.removed.length, markedCache.get(older, "k") !== undefined], [1, 0, true]);
|
||||
|
||||
// 2. The removal goes to the store of the record that exists, not the deleting type's; there the write form is false (it would
|
||||
// push to the wrong store) while the removal succeeds.
|
||||
const olderStore = recording([await heldIn("union-surface.msgpack")]);
|
||||
const newerStore = recording([]);
|
||||
const split = CultCache.builder().withDocumentType(older).withDocumentType(newer)
|
||||
.withBackingStore(olderStore.store, older).withBackingStore(newerStore.store, newer).build();
|
||||
await split.pullAllBackingStores();
|
||||
assert.equal(split.isWritable(newer, "k"), false, "isWritable answers the write form: the rename cannot replace a record in another store");
|
||||
assert.equal(await split.delete(newer, "k"), true);
|
||||
assert.deepEqual([olderStore.removed, newerStore.removed, split.get(older, "k")], [["k"], [], undefined]);
|
||||
|
||||
// 3. The held type's name and index lookups are cleared with the record.
|
||||
const single = recording([await heldIn("union-surface.msgpack")]);
|
||||
const cache = CultCache.builder().withDocumentType(older).withDocumentType(newer).withGenericStore(single.store).build();
|
||||
await cache.pullAllBackingStores();
|
||||
assert.equal(cache.getKeyByName(older, "surface"), "k");
|
||||
assert.equal(cache.getKeyByIndex(older, "byVersion", "3"), "k");
|
||||
await cache.delete(newer, "k");
|
||||
assert.equal(cache.getKeyByName(older, "surface"), undefined);
|
||||
assert.equal(cache.getKeyByIndex(older, "byVersion", "3"), undefined);
|
||||
assert.equal(cache.getAll(older).length, 0);
|
||||
});
|
||||
|
||||
test("a global type at a key other than the global key: isWritable is false and delete is the plain false of an unheld key", async () => {
|
||||
const global = defineDocumentType({ type: "tests.surface_global", schemaId: "tests.surface_global.v1", schemaName: "tests.surface_global", schemaVersion: "tests.surface_global.v1", schema: surfaceSchema, codec: surfaceCodec, name: "id", global: true });
|
||||
const cache = CultCache.builder().withDocumentType(global).withGenericStore(memoryStore([])).build();
|
||||
assert.equal(cache.isWritable(global, "x"), false);
|
||||
assert.equal(await cache.delete(global, "x"), false);
|
||||
});
|
||||
|
|
@ -670,7 +670,7 @@ test("delete removes only the record at that type and key", async () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("pushAll writes onto the store as it is: it replaces by key and removes nothing, and a soft pushAll leaves an existing store alone", async () => {
|
||||
test("pushAll replaces the whole store, and a soft pushAll leaves an existing store alone", async () => {
|
||||
const directory = await scratch();
|
||||
try {
|
||||
const path = join(directory, "store.cc");
|
||||
|
|
@ -681,14 +681,7 @@ test("pushAll writes onto the store as it is: it replaces by key and removes not
|
|||
await store.pushAll([entryOf("b", "t", 2)], { soft: true });
|
||||
assert.deepEqual([...(await readFile(path))], [...before], "soft leaves an existing store byte for byte");
|
||||
await store.pushAll([entryOf("b", "t", 2)]);
|
||||
assert.deepEqual(await snapshotOf(store), ["t/a:1", "t/b:2"], "a record the batch does not name stays");
|
||||
await store.pushAll([entryOf("a", "t", 7)]);
|
||||
assert.deepEqual(await snapshotOf(store), ["t/a:7", "t/b:2"], "a record the batch names is replaced");
|
||||
await store.pushAll([]);
|
||||
assert.deepEqual(await snapshotOf(store), ["t/a:7", "t/b:2"], "an empty flush from a cache that holds nothing wipes nothing");
|
||||
const before2 = [...(await readFile(path))];
|
||||
await assert.rejects(() => store.pushAll([entryOf("a", "u", 1)]), SchemaConflictError);
|
||||
assert.deepEqual([...(await readFile(path))], before2, "a batch that retypes a held key is refused and the file is unchanged");
|
||||
assert.deepEqual(await snapshotOf(store), ["t/b:2"]);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { join, resolve } from "node:path";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { decode, encode } from "@msgpack/msgpack";
|
||||
|
|
@ -21,14 +22,190 @@ import {
|
|||
encodeDocumentPayload,
|
||||
} from "../src/document";
|
||||
import { SingleFileMessagePackBackingStore } from "../src/single-file-messagepack-backing-store";
|
||||
import type { AnyCultCacheDocumentDefinition, CultCacheEnvelope } from "../src/types";
|
||||
|
||||
import {
|
||||
CANARY, type Node, type Surface, golden, hex, node, outcome, read, surfaceCodec, surfaceDocument, surfaceSchema, vectors, envelopeOf, memoryStore,
|
||||
} from "./union-surface-fixture";
|
||||
import type { AnyCultCacheDocumentDefinition, CacheBackingStore, CultCacheEnvelope } from "../src/types";
|
||||
|
||||
const { array, boolean, bytes, float32, float64, int32, int64, lazy, list, nullable, slots, string, union, unit } = codecs;
|
||||
|
||||
// The names the C# reference gives the fixture's types (CultSchemaTypeNames.FromType), as union-surface.cc carries them.
|
||||
const CS = "GameCult.Caching.Tests.UnionParityVectorTests+";
|
||||
|
||||
// The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected
|
||||
// outcome is read from those committed files; only the value the payload must decode to is spelled here.
|
||||
function findVectors(from: string): string {
|
||||
for (let dir = from; ; dir = resolve(dir, "..")) {
|
||||
if (existsSync(join(dir, "tests", "vectors", "union-parity"))) return join(dir, "tests", "vectors", "union-parity");
|
||||
if (resolve(dir, "..") === dir) throw new Error("tests/vectors/union-parity not found above the test.");
|
||||
}
|
||||
}
|
||||
const vectors = findVectors(__dirname);
|
||||
const read = async (name: string) => new Uint8Array(await readFile(join(vectors, name)));
|
||||
|
||||
// The string the C# writer put inside every refusal payload, to prove a refusal never echoes the payload.
|
||||
const CANARY = "CANARY-union-golden-4c2a";
|
||||
|
||||
type Node =
|
||||
| { kind: "Box"; id: string; children: (Node | null)[] }
|
||||
| { kind: "Text"; id: string; content: string }
|
||||
| { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] }
|
||||
| { kind: "Image"; id: string; uri: string; alt: string }
|
||||
| { kind: "Wrap"; inner: Node | null; id: string };
|
||||
interface Composition { name: string; body: Node | null }
|
||||
type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" };
|
||||
// The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records.
|
||||
type GoldenOutcome =
|
||||
| { kind: "Pending"; message: string; id: string }
|
||||
| { kind: "Completed"; sourceVersion: number; id: string }
|
||||
| { kind: "Denied"; id: string }
|
||||
| { kind: "Floor"; id: string }
|
||||
| { kind: "Ceiling"; id: string };
|
||||
type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number };
|
||||
type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number };
|
||||
interface Surface {
|
||||
id: string;
|
||||
version: number;
|
||||
root: Node | null;
|
||||
outcomes: (GoldenOutcome | null)[];
|
||||
part: Item | null;
|
||||
crafted: CraftedItem | null;
|
||||
missing: Node | null;
|
||||
}
|
||||
|
||||
const composition = slots<Composition>({ 0: ["name", string], 1: ["body", lazy(() => node)] }, `${CS}Composition`);
|
||||
const node: Codec<Node | null> = union<Node>("Node", {
|
||||
0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] })],
|
||||
1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] })],
|
||||
2: ["Use", slots<{ id: string; composition: Composition; slots: (Node | null)[] }>({
|
||||
0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))],
|
||||
})],
|
||||
5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })],
|
||||
6: ["Wrap", slots<{ inner: Node | null; id: string }>({ 0: ["inner", lazy(() => node)], 1: ["id", string] })],
|
||||
}, `${CS}Node`);
|
||||
const outcome = union<Outcome>("Outcome", {
|
||||
0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })],
|
||||
1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", int32] })],
|
||||
2: ["Denied", unit],
|
||||
[-2147483648]: ["Floor", unit],
|
||||
2147483647: ["Ceiling", unit],
|
||||
}, `${CS}Outcome`);
|
||||
const idOnly = slots<{ id: string }>({ 0: ["id", string] });
|
||||
const goldenOutcome = union<GoldenOutcome>("Outcome", {
|
||||
0: ["Pending", slots<{ message: string; id: string }>({ 0: ["message", string], 1: ["id", string] })],
|
||||
1: ["Completed", slots<{ sourceVersion: number; id: string }>({ 0: ["sourceVersion", int32], 1: ["id", string] })],
|
||||
2: ["Denied", idOnly],
|
||||
[-2147483648]: ["Floor", idOnly],
|
||||
2147483647: ["Ceiling", idOnly],
|
||||
}, `${CS}Outcome`);
|
||||
const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] });
|
||||
const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] });
|
||||
const item = union<Item>("Item", {
|
||||
0: ["Gem", slots<{ name: string }>({ 0: ["name", string] })],
|
||||
1: ["Tool", toolArm],
|
||||
2: ["Part", partArm],
|
||||
}, `${CS}Item`);
|
||||
const craftedItem = union<CraftedItem>("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }, `${CS}CraftedItem`);
|
||||
const surfaceCodec = slots<Surface>({
|
||||
0: ["id", string],
|
||||
1: ["version", int32],
|
||||
2: ["root", node],
|
||||
3: ["outcomes", list(goldenOutcome)],
|
||||
4: ["part", item],
|
||||
5: ["crafted", craftedItem],
|
||||
6: ["missing", node, null],
|
||||
}, `${CS}UnionSurface`);
|
||||
|
||||
const nodeSchema: z.ZodType<Node | null> = z.lazy(() =>
|
||||
z.union([
|
||||
z.null(),
|
||||
z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Box"), id: z.string(), children: z.array(nodeSchema) }),
|
||||
z.object({ kind: z.literal("Text"), id: z.string(), content: z.string() }),
|
||||
z.object({
|
||||
kind: z.literal("Use"),
|
||||
id: z.string(),
|
||||
composition: z.object({ name: z.string(), body: nodeSchema }),
|
||||
slots: z.array(nodeSchema),
|
||||
}),
|
||||
z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }),
|
||||
z.object({ kind: z.literal("Wrap"), inner: nodeSchema, id: z.string() }),
|
||||
]),
|
||||
]));
|
||||
const surfaceSchema = z.object({
|
||||
id: z.string(),
|
||||
version: z.number(),
|
||||
root: nodeSchema,
|
||||
outcomes: z.array(z.union([
|
||||
z.null(),
|
||||
z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Pending"), message: z.string(), id: z.string() }),
|
||||
z.object({ kind: z.literal("Completed"), sourceVersion: z.number(), id: z.string() }),
|
||||
z.object({ kind: z.literal("Denied"), id: z.string() }),
|
||||
z.object({ kind: z.literal("Floor"), id: z.string() }),
|
||||
z.object({ kind: z.literal("Ceiling"), id: z.string() }),
|
||||
]),
|
||||
])),
|
||||
part: z.union([z.null(), z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Gem"), name: z.string() }),
|
||||
z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }),
|
||||
z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }),
|
||||
])]),
|
||||
crafted: z.union([z.null(), z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }),
|
||||
z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }),
|
||||
])]),
|
||||
missing: nodeSchema,
|
||||
}) as unknown as z.ZodType<Surface>;
|
||||
|
||||
function surfaceDocument(schemaId?: string) {
|
||||
return defineDocumentType({
|
||||
type: "tests.union_surface",
|
||||
schemaId: schemaId ?? "tests.union_surface.v1",
|
||||
schemaName: "tests.union_surface",
|
||||
schemaVersion: "tests.union_surface.v1",
|
||||
schema: surfaceSchema,
|
||||
codec: surfaceCodec,
|
||||
name: "id",
|
||||
});
|
||||
}
|
||||
|
||||
const golden: Surface = {
|
||||
id: "surface",
|
||||
version: 3,
|
||||
root: {
|
||||
kind: "Box",
|
||||
id: "0000000000c0",
|
||||
children: [
|
||||
{ kind: "Text", id: "0000000000c1", content: "hello" },
|
||||
{
|
||||
kind: "Use",
|
||||
id: "0000000000c2",
|
||||
composition: {
|
||||
name: "comp",
|
||||
body: {
|
||||
kind: "Box",
|
||||
id: "0000000000d0",
|
||||
children: [
|
||||
{ kind: "Text", id: "0000000000d1", content: "x" },
|
||||
{ kind: "Image", id: "0000000000d2", uri: "uri://a", alt: "alt" },
|
||||
],
|
||||
},
|
||||
},
|
||||
slots: [{ kind: "Text", id: "0000000000e1", content: "slot" }, null, { kind: "Box", id: "0000000000e2", children: [] }],
|
||||
},
|
||||
{ kind: "Image", id: "0000000000c3", uri: "uri://b", alt: "" },
|
||||
],
|
||||
},
|
||||
outcomes: [
|
||||
{ kind: "Pending", message: "wait", id: "0000000000a1" },
|
||||
{ kind: "Completed", sourceVersion: 7, id: "0000000000a2" },
|
||||
{ kind: "Denied", id: "0000000000a3" },
|
||||
],
|
||||
part: { kind: "Tool", name: "hammer", uses: 12 },
|
||||
crafted: { kind: "Part", name: "gear", count: 4 },
|
||||
missing: null,
|
||||
};
|
||||
|
||||
const hex = (data: Uint8Array) => Buffer.from(data).toString("hex");
|
||||
|
||||
async function outcomeEntries() {
|
||||
const entries = decode(await read("outcomes.msgpack")) as [string, string, string, number | null, string][];
|
||||
return entries.map(([file, outcome, unionName, key, detail]) => ({ file, outcome, unionName, key, detail }));
|
||||
|
|
@ -338,7 +515,7 @@ test("the wire reader keeps 64-bit integers whole, tells a float from an integer
|
|||
assert.throws(() => readWire(fromHex("dd")), /truncated/u);
|
||||
assert.throws(() => readWire(fromHex("ddffffffff")), /truncated/u, "a huge array header reserves nothing");
|
||||
assert.throws(() => readWire(fromHex("a5616263")), /truncated/u);
|
||||
assert.equal(readWire(fromHex("0102")), 1, "the reader reads one value; the document owner refuses what follows");
|
||||
assert.throws(() => readWire(fromHex("0102")), /bytes after its value/u);
|
||||
assert.throws(() => readWire(fromHex("81a161a162")), /map, extension or reserved/u);
|
||||
assert.throws(() => readWire(fromHex("c1")), /map, extension or reserved/u);
|
||||
assert.throws(() => readWire(fromHex("d4010a")), /map, extension or reserved/u);
|
||||
|
|
@ -552,6 +729,19 @@ test("the Int32 edges of an arm key: the largest is an arm, one past either end
|
|||
assert.throws(() => edge.decode(readWire(fromHex("92d3ffffffff8000000090"))), { message: "Union U: unknown arm key -2147483648 (declared keys: 2147483647)." });
|
||||
});
|
||||
|
||||
function envelopeOf(payload: Uint8Array): CultCacheEnvelope {
|
||||
return { key: "surface", type: "tests.union_surface", payload: documentPayloadFromBytes(payload), storedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } {
|
||||
return {
|
||||
entries,
|
||||
async pullAll() { return entries.map((entry) => ({ ...entry })); },
|
||||
async push(entry) { entries.push(entry); },
|
||||
async delete() {},
|
||||
};
|
||||
}
|
||||
|
||||
test("arm keys span Int32 and are listed ascending; slot keys are non-negative Int32", () => {
|
||||
const keys = union<{ kind: "High" | "Zero" | "Neg" | "Low" }>("U", {
|
||||
2147483647: ["High", unit],
|
||||
|
|
@ -725,114 +915,3 @@ test("a cache load refuses a store whose record is past the limit, naming the ty
|
|||
const refusing = CultCache.builder().withDocumentType(document).withGenericStore(memoryStore([envelopeOf(await read("depth-past-limit.msgpack"))])).build();
|
||||
await assert.rejects(() => refusing.pullAllBackingStores(), (error: Error) => error instanceof CultCachePayloadDepthError && !error.message.includes(CANARY));
|
||||
});
|
||||
|
||||
// ---- bytes-after vectors: a payload is exactly one MessagePack value (tests/vectors/union-parity/README.md) ----
|
||||
|
||||
// The C# reference writes the vectors: union-surface.msgpack followed by a tail the first value does not own. The one
|
||||
// owner refuses by count, naming the type, from the walk's end; it never walks or echoes the tail.
|
||||
async function bytesAfterEntries() {
|
||||
const raw = decode(await read("bytes-after-outcomes.msgpack")) as unknown[][];
|
||||
return raw.map(([file, outcome, unionName, key, detail]) => ({ file: file as string, outcome: outcome as string, unionName: unionName as string, key: key as number | null, detail: detail as string }));
|
||||
}
|
||||
|
||||
// The C# vector writer's canary (BytesAfterVectorTests): it sits in the deep tail, so an echo of the tail would carry it.
|
||||
const TAIL_CANARY = "CANARY-bytes-after-7d1e";
|
||||
const valueLength = async () => (await read("union-surface.msgpack")).length;
|
||||
const plainDocument = () => defineDocumentType({ type: "tests.plain_surface", schema: z.unknown() });
|
||||
const codecDocument = () => defineDocumentType({ type: "tests.codec_surface", schema: surfaceSchema, codec: surfaceCodec });
|
||||
|
||||
async function assertBytesAfterRefused(file: string, tail: number, run: () => unknown | Promise<unknown>, type: string) {
|
||||
const clause = `${type}`;
|
||||
await assert.rejects(
|
||||
async () => run(),
|
||||
(error: Error) => {
|
||||
assert.ok(!(error instanceof CultCachePayloadDepthError), `${file}: the tail was walked`);
|
||||
assert.ok(!error.message.includes(CANARY) && !error.message.includes(TAIL_CANARY), `${file}: the refusal echoed the tail`);
|
||||
assert.ok(!error.message.includes("the limit is"), `${file}: the depth refusal`);
|
||||
assert.ok(error.message.includes(clause), `${file}: names the type: ${error.message}`);
|
||||
assert.ok(error.message.includes(`${tail} bytes after its first MessagePack value`), `${file}: names the count: ${error.message}`);
|
||||
return true;
|
||||
},
|
||||
file,
|
||||
);
|
||||
}
|
||||
|
||||
test("every bytes-after vector is refused by the one owner, by count, on the codec, plain and store paths", async () => {
|
||||
const entries = await bytesAfterEntries();
|
||||
assert.equal(entries.length, 4);
|
||||
const length = await valueLength();
|
||||
const surface = surfaceDocument();
|
||||
const plain = plainDocument();
|
||||
const codecOnly = codecDocument();
|
||||
const dir = await mkdtemp(join(tmpdir(), "bytes-after-"));
|
||||
for (const { file, outcome, unionName, detail } of entries) {
|
||||
assert.equal(outcome, "refused");
|
||||
assert.equal(unionName, "UnionSurface");
|
||||
assert.equal(detail, "bytes-after");
|
||||
const bytes = await read(file);
|
||||
const tail = bytes.length - length;
|
||||
assert.ok(tail > 0, file);
|
||||
if (file === "bytes-after-deep-tail.msgpack") assert.ok(Buffer.from(bytes).includes(TAIL_CANARY), "the deep tail carries the canary the assertions look for");
|
||||
const payload = documentPayloadFromBytes(bytes);
|
||||
await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(surface, payload), surface.type);
|
||||
await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(plain, payload), plain.type);
|
||||
await assertBytesAfterRefused(file, tail, () => decodeDocumentPayload(codecOnly, payload), codecOnly.type);
|
||||
|
||||
// The store path: a single-file store holding the vector as a record payload refuses on load.
|
||||
const path = join(dir, `${file}.cc`);
|
||||
await new SingleFileMessagePackBackingStore(path).push(envelopeOf(bytes));
|
||||
const cache = CultCache.builder().withDocumentType(surface).withGenericStore(new SingleFileMessagePackBackingStore(path)).build();
|
||||
await assertBytesAfterRefused(file, tail, () => cache.pullAllBackingStores(), surface.type);
|
||||
|
||||
// The put path: an envelope arriving with the tail is refused and held nowhere.
|
||||
const written: CultCacheEnvelope[] = [];
|
||||
const held = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build();
|
||||
await assertBytesAfterRefused(file, tail, () => held.putEnvelope(surface, envelopeOf(bytes)), surface.type);
|
||||
assert.equal(written.length, 0, `${file}: nothing reached the store`);
|
||||
assert.equal(held.get(surface, "surface"), undefined, `${file}: nothing is held in memory`);
|
||||
}
|
||||
});
|
||||
|
||||
test("a payload with an unread slot and a tail is refused by count before any slot is marked or the record held", async () => {
|
||||
const withSlot = await read("extra-slot-record.msgpack");
|
||||
const surface = surfaceDocument();
|
||||
const tailed = Uint8Array.from([...withSlot, 0xc0]);
|
||||
const whole = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(withSlot)])).build();
|
||||
await whole.pullAllBackingStores();
|
||||
assert.ok(whole.unreadSlots(surface, "surface"), "without a tail the record is held with its unread slots");
|
||||
|
||||
const loading = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore([envelopeOf(tailed)])).build();
|
||||
await assertBytesAfterRefused("extra-slot-record+tail", 1, () => loading.pullAllBackingStores(), surface.type);
|
||||
assert.equal(loading.get(surface, "surface"), undefined);
|
||||
assert.equal(loading.unreadSlots(surface, "surface"), undefined);
|
||||
|
||||
const written: CultCacheEnvelope[] = [];
|
||||
const putting = CultCache.builder().withDocumentType(surface).withGenericStore(memoryStore(written)).build();
|
||||
await assertBytesAfterRefused("extra-slot-record+tail", 1, () => putting.putEnvelope(surface, envelopeOf(tailed)), surface.type);
|
||||
assert.equal(written.length, 0);
|
||||
assert.equal(putting.get(surface, "surface"), undefined);
|
||||
assert.equal(putting.unreadSlots(surface, "surface"), undefined);
|
||||
});
|
||||
|
||||
test("a payload that ends early is not bytes after its value: the decoder refuses it, and a user formatter's bytes are not walked", async () => {
|
||||
const whole = await read("union-surface.msgpack");
|
||||
const surface = surfaceDocument();
|
||||
for (const cut of [0, 1, 10, whole.length - 1]) {
|
||||
assert.throws(
|
||||
() => decodeDocumentPayload(surface, documentPayloadFromBytes(whole.slice(0, cut))),
|
||||
(error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError),
|
||||
`truncated at ${cut}`,
|
||||
);
|
||||
}
|
||||
// A header cut short of its length is an early end too, whatever width it declares.
|
||||
for (const early of ["dc00", "dd000000", "de00", "c4", "d9", "a5616263", "9291"]) {
|
||||
assert.throws(
|
||||
() => decodeDocumentPayload(surface, documentPayloadFromBytes(fromHex(early))),
|
||||
(error: Error) => !error.message.includes("bytes after") && !(error instanceof CultCachePayloadDepthError),
|
||||
`early end ${early}`,
|
||||
);
|
||||
}
|
||||
const opaque = defineDocumentType({ type: "tests.opaque", schema: z.unknown(), formatter: { encode: (value: Uint8Array) => value, decode: (bytes: Uint8Array) => bytes } });
|
||||
const tailed = await read("bytes-after-nil.msgpack");
|
||||
assert.equal((decodeDocumentPayload(opaque, documentPayloadFromBytes(tailed)) as Uint8Array).length, tailed.length);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,240 +0,0 @@
|
|||
import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { codecs } from "../src";
|
||||
import type { Codec } from "../src/codec";
|
||||
import { defineDocumentType, documentPayloadFromBytes } from "../src/document";
|
||||
import type { CacheBackingStore, CultCacheEnvelope } from "../src/types";
|
||||
|
||||
// The union-surface fixture: the codecs, schema and value tests/vectors/union-parity was written from, shared by the
|
||||
// union-parity walker and the slot-readonly walker so the shape is spelled once.
|
||||
const { list, int32, lazy, slots, string, union, unit } = codecs;
|
||||
|
||||
|
||||
// The names the C# reference gives the fixture's types (CultSchemaTypeNames.FromType), as union-surface.cc carries them.
|
||||
export const CS = "GameCult.Caching.Tests.UnionParityVectorTests+";
|
||||
|
||||
// The golden is tests/vectors/union-parity, written by the C# reference. Everything below that is bytes or an expected
|
||||
// outcome is read from those committed files; only the value the payload must decode to is spelled here.
|
||||
export function findVectors(from: string): string {
|
||||
for (let dir = from; ; dir = resolve(dir, "..")) {
|
||||
if (existsSync(join(dir, "tests", "vectors", "union-parity"))) return join(dir, "tests", "vectors", "union-parity");
|
||||
if (resolve(dir, "..") === dir) throw new Error("tests/vectors/union-parity not found above the test.");
|
||||
}
|
||||
}
|
||||
export const vectors = findVectors(__dirname);
|
||||
export const read = async (name: string) => new Uint8Array(await readFile(join(vectors, name)));
|
||||
|
||||
// The string the C# writer put inside every refusal payload, to prove a refusal never echoes the payload.
|
||||
export const CANARY = "CANARY-union-golden-4c2a";
|
||||
|
||||
export type Node =
|
||||
| { kind: "Box"; id: string; children: (Node | null)[] }
|
||||
| { kind: "Text"; id: string; content: string }
|
||||
| { kind: "Use"; id: string; composition: Composition; slots: (Node | null)[] }
|
||||
| { kind: "Image"; id: string; uri: string; alt: string }
|
||||
| { kind: "Wrap"; inner: Node | null; id: string };
|
||||
export interface Composition { name: string; body: Node | null }
|
||||
export type Outcome = { kind: "Pending"; message: string } | { kind: "Completed"; sourceVersion: number } | { kind: "Denied" } | { kind: "Floor" } | { kind: "Ceiling" };
|
||||
// The golden's Outcome: the arms of a list carry an element id, so Denied, Floor and Ceiling are one-slot records.
|
||||
export type GoldenOutcome =
|
||||
| { kind: "Pending"; message: string; id: string }
|
||||
| { kind: "Completed"; sourceVersion: number; id: string }
|
||||
| { kind: "Denied"; id: string }
|
||||
| { kind: "Floor"; id: string }
|
||||
| { kind: "Ceiling"; id: string };
|
||||
export type Item = { kind: "Gem"; name: string } | { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number };
|
||||
export type CraftedItem = { kind: "Tool"; name: string; uses: number } | { kind: "Part"; name: string; count: number };
|
||||
export interface Surface {
|
||||
id: string;
|
||||
version: number;
|
||||
root: Node | null;
|
||||
outcomes: (GoldenOutcome | null)[];
|
||||
part: Item | null;
|
||||
crafted: CraftedItem | null;
|
||||
missing: Node | null;
|
||||
}
|
||||
|
||||
export const composition = slots<Composition>({ 0: ["name", string], 1: ["body", lazy(() => node)] }, `${CS}Composition`);
|
||||
export const node: Codec<Node | null> = union<Node>("Node", {
|
||||
0: ["Box", slots<{ id: string; children: (Node | null)[] }>({ 0: ["id", string], 1: ["children", list(lazy(() => node))] }, `${CS}Box`)],
|
||||
1: ["Text", slots<{ id: string; content: string }>({ 0: ["id", string], 1: ["content", string] }, `${CS}Text`)],
|
||||
2: ["Use", slots<{ id: string; composition: Composition; slots: (Node | null)[] }>({
|
||||
0: ["id", string], 1: ["composition", composition], 2: ["slots", list(lazy(() => node))],
|
||||
})],
|
||||
5: ["Image", slots<{ id: string; uri: string; alt: string }>({ 0: ["id", string], 1: ["uri", string], 3: ["alt", string] })],
|
||||
6: ["Wrap", slots<{ inner: Node | null; id: string }>({ 0: ["inner", lazy(() => node)], 1: ["id", string] })],
|
||||
}, `${CS}Node`);
|
||||
export const outcome = union<Outcome>("Outcome", {
|
||||
0: ["Pending", slots<{ message: string }>({ 0: ["message", string] })],
|
||||
1: ["Completed", slots<{ sourceVersion: number }>({ 0: ["sourceVersion", int32] })],
|
||||
2: ["Denied", unit],
|
||||
[-2147483648]: ["Floor", unit],
|
||||
2147483647: ["Ceiling", unit],
|
||||
}, `${CS}Outcome`);
|
||||
export const idOnly = slots<{ id: string }>({ 0: ["id", string] });
|
||||
export const goldenOutcome = union<GoldenOutcome>("Outcome", {
|
||||
0: ["Pending", slots<{ message: string; id: string }>({ 0: ["message", string], 1: ["id", string] })],
|
||||
1: ["Completed", slots<{ sourceVersion: number; id: string }>({ 0: ["sourceVersion", int32], 1: ["id", string] })],
|
||||
2: ["Denied", idOnly],
|
||||
[-2147483648]: ["Floor", idOnly],
|
||||
2147483647: ["Ceiling", idOnly],
|
||||
}, `${CS}Outcome`);
|
||||
export const toolArm = slots<{ name: string; uses: number }>({ 0: ["name", string], 1: ["uses", int32] });
|
||||
export const partArm = slots<{ name: string; count: number }>({ 0: ["name", string], 1: ["count", int32] });
|
||||
export const item = union<Item>("Item", {
|
||||
0: ["Gem", slots<{ name: string }>({ 0: ["name", string] })],
|
||||
1: ["Tool", toolArm],
|
||||
2: ["Part", partArm],
|
||||
}, `${CS}Item`);
|
||||
export const craftedItem = union<CraftedItem>("CraftedItem", { 0: ["Tool", toolArm], 1: ["Part", partArm] }, `${CS}CraftedItem`);
|
||||
export const surfaceCodecOf = (typeName: string) => slots<Surface>({
|
||||
0: ["id", string],
|
||||
1: ["version", int32],
|
||||
2: ["root", node],
|
||||
3: ["outcomes", list(goldenOutcome)],
|
||||
4: ["part", item],
|
||||
5: ["crafted", craftedItem],
|
||||
6: ["missing", node, null],
|
||||
}, typeName);
|
||||
export const surfaceCodec = surfaceCodecOf(`${CS}UnionSurface`);
|
||||
|
||||
export const nodeSchema: z.ZodType<Node | null> = z.lazy(() =>
|
||||
z.union([
|
||||
z.null(),
|
||||
z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Box"), id: z.string(), children: z.array(nodeSchema) }),
|
||||
z.object({ kind: z.literal("Text"), id: z.string(), content: z.string() }),
|
||||
z.object({
|
||||
kind: z.literal("Use"),
|
||||
id: z.string(),
|
||||
composition: z.object({ name: z.string(), body: nodeSchema }),
|
||||
slots: z.array(nodeSchema),
|
||||
}),
|
||||
z.object({ kind: z.literal("Image"), id: z.string(), uri: z.string(), alt: z.string() }),
|
||||
z.object({ kind: z.literal("Wrap"), inner: nodeSchema, id: z.string() }),
|
||||
]),
|
||||
]));
|
||||
export const surfaceSchema = z.object({
|
||||
id: z.string(),
|
||||
version: z.number(),
|
||||
root: nodeSchema,
|
||||
outcomes: z.array(z.union([
|
||||
z.null(),
|
||||
z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Pending"), message: z.string(), id: z.string() }),
|
||||
z.object({ kind: z.literal("Completed"), sourceVersion: z.number(), id: z.string() }),
|
||||
z.object({ kind: z.literal("Denied"), id: z.string() }),
|
||||
z.object({ kind: z.literal("Floor"), id: z.string() }),
|
||||
z.object({ kind: z.literal("Ceiling"), id: z.string() }),
|
||||
]),
|
||||
])),
|
||||
part: z.union([z.null(), z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Gem"), name: z.string() }),
|
||||
z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }),
|
||||
z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }),
|
||||
])]),
|
||||
crafted: z.union([z.null(), z.discriminatedUnion("kind", [
|
||||
z.object({ kind: z.literal("Tool"), name: z.string(), uses: z.number() }),
|
||||
z.object({ kind: z.literal("Part"), name: z.string(), count: z.number() }),
|
||||
])]),
|
||||
missing: nodeSchema,
|
||||
}) as unknown as z.ZodType<Surface>;
|
||||
|
||||
// The reference's SurfaceDeclaring: the same layout, schema tests.union_surface.v2, listing the older wide id compatible.
|
||||
export function surfaceDeclaringDocument() {
|
||||
return defineDocumentType({
|
||||
type: "tests.union_surface",
|
||||
schemaId: "tests.union_surface.v2",
|
||||
schemaName: "tests.union_surface",
|
||||
schemaVersion: "tests.union_surface.v2",
|
||||
compatibleSchemaIds: ["tests.union_surface.wide"],
|
||||
schema: surfaceSchema,
|
||||
codec: surfaceCodecOf(`${CS}SurfaceDeclaring`),
|
||||
name: "id",
|
||||
});
|
||||
}
|
||||
|
||||
export function surfaceDocument(schemaId?: string, contentHash?: string) {
|
||||
return defineDocumentType({
|
||||
...(contentHash === undefined ? {} : { contentHash }),
|
||||
type: "tests.union_surface",
|
||||
schemaId: schemaId ?? "tests.union_surface.v1",
|
||||
schemaName: "tests.union_surface",
|
||||
schemaVersion: "tests.union_surface.v1",
|
||||
schema: surfaceSchema,
|
||||
codec: surfaceCodec,
|
||||
name: "id",
|
||||
});
|
||||
}
|
||||
|
||||
export const golden: Surface = {
|
||||
id: "surface",
|
||||
version: 3,
|
||||
root: {
|
||||
kind: "Box",
|
||||
id: "0000000000c0",
|
||||
children: [
|
||||
{ kind: "Text", id: "0000000000c1", content: "hello" },
|
||||
{
|
||||
kind: "Use",
|
||||
id: "0000000000c2",
|
||||
composition: {
|
||||
name: "comp",
|
||||
body: {
|
||||
kind: "Box",
|
||||
id: "0000000000d0",
|
||||
children: [
|
||||
{ kind: "Text", id: "0000000000d1", content: "x" },
|
||||
{ kind: "Image", id: "0000000000d2", uri: "uri://a", alt: "alt" },
|
||||
],
|
||||
},
|
||||
},
|
||||
slots: [{ kind: "Text", id: "0000000000e1", content: "slot" }, null, { kind: "Box", id: "0000000000e2", children: [] }],
|
||||
},
|
||||
{ kind: "Image", id: "0000000000c3", uri: "uri://b", alt: "" },
|
||||
],
|
||||
},
|
||||
outcomes: [
|
||||
{ kind: "Pending", message: "wait", id: "0000000000a1" },
|
||||
{ kind: "Completed", sourceVersion: 7, id: "0000000000a2" },
|
||||
{ kind: "Denied", id: "0000000000a3" },
|
||||
],
|
||||
part: { kind: "Tool", name: "hammer", uses: 12 },
|
||||
crafted: { kind: "Part", name: "gear", count: 4 },
|
||||
missing: null,
|
||||
};
|
||||
export const hex = (data: Uint8Array) => Buffer.from(data).toString("hex");
|
||||
|
||||
// tests.slot_unit: a document [Name, Outcome: UnitOutcome], whose one arm Denied2 is a unit arm (extra-slot-store.cc, record
|
||||
// unit-arm). The reader's names are the reference's: a mark names Denied2.
|
||||
export const unitOutcome = union<{ kind: "Denied2" }>("UnitOutcome", { 2: ["Denied2", unit] }, `${CS}UnitOutcome`);
|
||||
export function slotUnitDocument(schemaId: string, contentHash?: string) {
|
||||
return defineDocumentType({
|
||||
...(contentHash === undefined ? {} : { contentHash }),
|
||||
type: "tests.slot_unit",
|
||||
schemaId,
|
||||
schemaName: "tests.slot_unit",
|
||||
schemaVersion: "tests.slot_unit.v1",
|
||||
schema: z.object({ name: z.string(), outcome: z.union([z.null(), z.object({ kind: z.literal("Denied2") })]) }),
|
||||
codec: slots<{ name: string; outcome: { kind: "Denied2" } | null }>({ 0: ["name", string], 1: ["outcome", unitOutcome] }, `${CS}SlotUnit`),
|
||||
name: "name",
|
||||
});
|
||||
}
|
||||
|
||||
export function envelopeOf(payload: Uint8Array): CultCacheEnvelope {
|
||||
return { key: "surface", type: "tests.union_surface", payload: documentPayloadFromBytes(payload), storedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
export function memoryStore(entries: CultCacheEnvelope[]): CacheBackingStore & { entries: CultCacheEnvelope[] } {
|
||||
return {
|
||||
entries,
|
||||
async pullAll() { return entries.map((entry) => ({ ...entry })); },
|
||||
async push(entry) { entries.push(entry); },
|
||||
async delete() {},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -9,10 +9,6 @@ All notable changes to `@gamecult/cultmesh-browser` are recorded here, following
|
|||
|
||||
- The package is renamed `cultmesh-browser` to `@gamecult/cultmesh-browser` (the repo already scopes `@gamecult/cultcache-ts`), so the unscoped name cannot be claimed by someone else. Import from `@gamecult/cultmesh-browser` and its subpaths; the API is unchanged. Depends on `@gamecult/cultnet-ts`; the interop test uses `@gamecult/cultmesh-ts`.
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Needs `@gamecult/cultnet-ts` `^0.4.0` (was `^0.3.0`), moving with the `cultnet-ts` 0.4.0 release.
|
||||
|
||||
## [0.4.0]
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@gamecult/cultcache-ts": "^0.15.0",
|
||||
"@gamecult/cultnet-ts": "^0.4.0"
|
||||
"@gamecult/cultnet-ts": "^0.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.0",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ classifiers = [
|
|||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
]
|
||||
dependencies = ["cultcache-py>=0.4.0", "cultnet-py>=0.1.0", "cryptography>=41"]
|
||||
dependencies = ["cultcache-py>=0.2.0", "cultnet-py>=0.1.0", "cryptography>=41"]
|
||||
[project.scripts]
|
||||
cultmesh-py-daemon = "cultmesh_py.daemon:main"
|
||||
cultmesh-py-verify = "cultmesh_py.verify:main"
|
||||
|
|
|
|||
|
|
@ -180,21 +180,6 @@ class RemoteWriteAuthorityTests(unittest.TestCase):
|
|||
reply = self.send(granting, unowned)
|
||||
self.assertEqual(reply[0]["code"], "unregistered_document_put", "a granted put still meets the store's refusal")
|
||||
|
||||
def test_a_put_with_a_payload_tail_and_an_unknown_schema_id_is_refused_whole_under_the_canonical_id(self) -> None:
|
||||
# The payload head names the registered schema, so the put routes by stamp; the tail makes the payload more
|
||||
# than one MessagePack value. It is answered message_invalid (not unregistered_document_put), nothing is
|
||||
# stored, and the authority is asked under the canonical id, not the id as sent.
|
||||
authority = Recording(lambda _: CultMeshRemoteWriteDecision.grant())
|
||||
server = self.build(authority)
|
||||
tailed = put_of(KEY_A, "x", schema_id="sha256:nobody-owns-this")
|
||||
tailed["document"]["payload"] = bytes(tailed["document"]["payload"]) + b"\x01"
|
||||
|
||||
replies = self.send(server, tailed)
|
||||
|
||||
self.assertEqual([(reply["code"], reply["details"]["field"]) for reply in replies], [("message_invalid", "document")])
|
||||
self.assertIsNone(self.text_of(server, KEY_A))
|
||||
self.assertEqual([request.schema_id for request in authority.requests], [SCHEMA_ID])
|
||||
|
||||
def test_an_authority_that_raises_refuses_and_is_logged_as_an_error(self) -> None:
|
||||
def explode(_: Any) -> Any:
|
||||
raise RuntimeError("authority exploded")
|
||||
|
|
|
|||
|
|
@ -735,34 +735,30 @@ impl CultMeshNode {
|
|||
self.apply_raw_snapshot_documents(documents)
|
||||
}
|
||||
|
||||
/// Applies the snapshot documents as one cache commit, as C# does: a refusal (a key this node holds read-only because
|
||||
/// of unread slots from a newer peer or a payload this build cannot read, or any other the cache judges) applies
|
||||
/// nothing. A schema id the registry does not know is skipped, except one with a `.vN` suffix, which is taken as
|
||||
/// registered and refuses the commit when its type is not (C# throws on an unknown schema too). The commit lands in
|
||||
/// one backing store, so a snapshot spanning stores is refused whole; C# `CultCache.Commit` refuses a cross-store batch too
|
||||
/// ("The batch spans A and B; a commit lands in one home store"), so the two agree.
|
||||
/// Applies each snapshot document of a registered schema with `put_raw_envelope`. The first refusal ends the pull
|
||||
/// and the documents before it stay applied; a key this node holds read-only (unread slots from a newer peer, or a
|
||||
/// payload this build cannot read) is refused like any other. C# applies a raw snapshot as one cache commit, so a
|
||||
/// refusal there applies nothing: this is a divergence, owed by follow-up mesh-snapshot-atomic (stage the pull through
|
||||
/// `put_prepared_batch`).
|
||||
fn apply_raw_snapshot_documents(
|
||||
&mut self,
|
||||
documents: Vec<cultnet_rs::CultNetRawDocumentRecord>,
|
||||
) -> Result<usize> {
|
||||
let mut staged = Vec::new();
|
||||
let mut applied = 0usize;
|
||||
for document in documents {
|
||||
let Some(document_type) =
|
||||
registered_document_type_for_schema(&self.documents, &document.schema_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
staged.push(CultCacheEnvelope {
|
||||
self.cache.put_raw_envelope(CultCacheEnvelope {
|
||||
key: document.record_key,
|
||||
r#type: document_type,
|
||||
payload: document.payload,
|
||||
stored_at: document.stored_at,
|
||||
schema_id: Some(document.schema_id),
|
||||
});
|
||||
}
|
||||
let applied = staged.len();
|
||||
if applied > 0 {
|
||||
self.cache.put_prepared_batch(staged)?;
|
||||
})?;
|
||||
applied += 1;
|
||||
}
|
||||
Ok(applied)
|
||||
}
|
||||
|
|
@ -1211,22 +1207,10 @@ mod tests {
|
|||
body: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, cultcache_rs::DatabaseEntry)]
|
||||
#[cultcache(type = "cultmesh.test.memo", schema = "CultMeshTestMemo")]
|
||||
struct Memo {
|
||||
#[cultcache(key = 0)]
|
||||
body: String,
|
||||
}
|
||||
|
||||
cultmesh_documents!(TestDocuments {
|
||||
Note => "cultmesh.test.note.v0",
|
||||
});
|
||||
|
||||
cultmesh_documents!(TwoTypeDocuments {
|
||||
Note => "cultmesh.test.note.v0",
|
||||
Memo => "cultmesh.test.memo.v0",
|
||||
});
|
||||
|
||||
fn raw_note(key: &str, payload: Vec<u8>) -> cultnet_rs::CultNetRawDocumentRecord {
|
||||
cultnet_rs::CultNetRawDocumentRecord {
|
||||
schema_id: "cultmesh.test.note.v0".to_string(),
|
||||
|
|
@ -1242,7 +1226,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn a_snapshot_document_for_a_read_only_key_refuses_the_whole_pull_as_csharp_does() -> Result<()> {
|
||||
fn a_snapshot_document_for_a_read_only_key_ends_the_pull_unlike_csharp_which_applies_nothing() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let mut node = CultMesh::create_node(temp.path().join("m.cc"), TestDocuments, Default::default())?;
|
||||
// A newer peer's note: one slot past the declared one.
|
||||
|
|
@ -1262,60 +1246,12 @@ mod tests {
|
|||
assert_eq!(conflict.record_key, "held");
|
||||
assert!(conflict.unread_slots.is_some());
|
||||
assert!(!error.to_string().contains("extra slot"));
|
||||
assert!(node.get::<Note>("before")?.is_none());
|
||||
assert_eq!(node.get_required::<Note>("before")?.body, "b");
|
||||
assert_eq!(node.get_required::<Note>("held")?.body, "newer body");
|
||||
assert!(node.get::<Note>("after")?.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_snapshot_spanning_two_backing_stores_is_refused_whole_and_applies_nothing() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let mut node = CultMesh::create_node(temp.path().join("m.cc"), TwoTypeDocuments, Default::default())?;
|
||||
// A node has one store through the public API; route Memo to a second one by hand (C# refuses the same commit).
|
||||
node.cache.add_backing_store(
|
||||
SingleFileMessagePackBackingStore::new(temp.path().join("memo.cc")),
|
||||
[Memo::TYPE],
|
||||
)?;
|
||||
node.cache.pull_all_backing_stores()?;
|
||||
let plain = |body: &str| rmp_serde::to_vec(&(body,));
|
||||
let memo = |key: &str, payload| {
|
||||
let mut record = raw_note(key, payload);
|
||||
record.schema_id = "cultmesh.test.memo.v0".to_string();
|
||||
record
|
||||
};
|
||||
let error = node
|
||||
.apply_raw_snapshot_documents(vec![
|
||||
raw_note("n", plain("note")?),
|
||||
memo("m", plain("memo")?),
|
||||
])
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("cannot span backing stores"), "{error}");
|
||||
assert!(node.get::<Note>("n")?.is_none());
|
||||
assert!(node.get::<Memo>("m")?.is_none());
|
||||
assert!(!temp.path().join("m.cc").exists() && !temp.path().join("memo.cc").exists());
|
||||
|
||||
// Each store alone still takes its own snapshot.
|
||||
assert_eq!(node.apply_raw_snapshot_documents(vec![memo("m", plain("memo")?)])?, 1);
|
||||
assert_eq!(node.get_required::<Memo>("m")?.body, "memo");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_snapshot_applied_to_a_stale_node_keeps_the_records_another_writer_added() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let path = temp.path().join("m.cc");
|
||||
let mut node = CultMesh::create_node(&path, TestDocuments, Default::default())?;
|
||||
let mut other = CultMesh::create_node(&path, TestDocuments, Default::default())?;
|
||||
other.put("added", &Note { body: "other writer".to_string() })?;
|
||||
let plain = |body: &str| rmp_serde::to_vec(&(body,));
|
||||
assert_eq!(node.apply_raw_snapshot_documents(vec![raw_note("snap", plain("s")?)])?, 1);
|
||||
let reopened = CultMesh::create_node(&path, TestDocuments, Default::default())?;
|
||||
assert_eq!(reopened.get_required::<Note>("added")?.body, "other writer");
|
||||
assert_eq!(reopened.get_required::<Note>("snap")?.body, "s");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_round_trips_registered_documents() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
|
|
|
|||
|
|
@ -29,8 +29,6 @@ All notable changes to `@gamecult/cultmesh-ts` are recorded here, following
|
|||
|
||||
- Needs the cultcache-ts release that ships with this change (its manifest range moves with that release): it imports `schemaIdentityOf` and `SchemaConflictError`, which 0.14.x does not export.
|
||||
|
||||
- Needs `@gamecult/cultnet-ts` `^0.4.0` (was `^0.3.0`): this package imports names that `cultnet-ts` 0.3.0 does not export (`cultNetMessageInvalidError`, `cultNetTextError`, `cultNetWriteNotAuthorizedError`, `WriteTracker`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `helloAdvertisesWriteReceipts`, `resultWindowMs`). The declared-range check (`npm run test:pack:ts`) found the old range admitting a release that lacks them.
|
||||
|
||||
## [0.2.0]
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@gamecult/cultcache-ts": "^0.15.0",
|
||||
"@gamecult/cultnet-ts": "^0.4.0",
|
||||
"@gamecult/cultnet-ts": "^0.3.0",
|
||||
"koffi": "3.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -3933,7 +3933,7 @@ export class CultMeshNode {
|
|||
}
|
||||
|
||||
public async flush(soft = false): Promise<void> {
|
||||
await this.store.pushAll?.(this.cache.storedSnapshot(), { soft });
|
||||
await this.store.pushAll?.(this.cache.snapshot(), { soft });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import test from "node:test";
|
|||
import assert from "node:assert/strict";
|
||||
import { decodeWireValue, encodeWireValue } from "@gamecult/cultnet-ts/wire";
|
||||
import { z } from "zod";
|
||||
import { SchemaConflictError, SingleFileMessagePackBackingStore, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts";
|
||||
import { SchemaConflictError, defineDocumentType, documentPayloadBytes, documentPayloadFromBytes } from "@gamecult/cultcache-ts";
|
||||
import {
|
||||
CultNetDocumentRegistry,
|
||||
CultNetPeer,
|
||||
|
|
@ -109,34 +109,6 @@ test("CultMesh TS opens a durable local node and registers document bindings", a
|
|||
assert.ok(reopened.documents.getBySchemaId("cultmesh.note.v0"));
|
||||
});
|
||||
|
||||
test("CultMesh TS flush from a node that never pulled leaves the records the store holds", async () => {
|
||||
const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-flush-")), "node.ccmp");
|
||||
const first = await CultMesh.startNode(filePath, { documents: [noteDocument] });
|
||||
await first.put(noteDocument, "note:held", { noteId: "note:held", body: "already in the store" });
|
||||
await first.flush();
|
||||
|
||||
const unpulled = await CultMesh.startNode(filePath, { documents: [noteDocument], pullOnStart: false });
|
||||
await unpulled.flush();
|
||||
|
||||
const reopened = await CultMesh.startNode(filePath, { documents: [noteDocument] });
|
||||
assert.equal(reopened.getRequired(noteDocument, "note:held").body, "already in the store");
|
||||
});
|
||||
|
||||
test("CultMesh TS flush of a node that loaded a legacy-key global leaves the store reopenable", async () => {
|
||||
const settings = defineDocumentType({ type: "cultmesh.settings", schema: z.object({ theme: z.string() }), global: true });
|
||||
const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-legacy-flush-")), "node.ccmp");
|
||||
const seed = await CultMesh.startNode(join(await mkdtemp(join(tmpdir(), "cultmesh-ts-seed-")), "seed.ccmp"), { documents: [settings] });
|
||||
await seed.cache.putGlobal(settings, { theme: "iron" });
|
||||
const legacy = { ...seed.cache.getRequiredEnvelope(settings, "__global__"), key: "legacy" };
|
||||
await new SingleFileMessagePackBackingStore(filePath).push(legacy);
|
||||
|
||||
const node = await CultMesh.startNode(filePath, { documents: [settings] });
|
||||
await node.flush();
|
||||
|
||||
const reopened = await CultMesh.startNode(filePath, { documents: [settings] });
|
||||
assert.deepEqual(reopened.cache.getRequiredGlobal(settings), { theme: "iron" });
|
||||
});
|
||||
|
||||
test("CultMesh TS document handles hide local cache plumbing behind typed reactive reads", async () => {
|
||||
const filePath = join(await mkdtemp(join(tmpdir(), "cultmesh-ts-doc-")), "node.ccmp");
|
||||
const node = await CultMesh.startNode(filePath, {
|
||||
|
|
|
|||
|
|
@ -83,16 +83,3 @@ test("a provider command that is not a map, or lacks a required field, is refuse
|
|||
(error: Error) => error.message === "CultMesh provider command must be a MessagePack map.",
|
||||
);
|
||||
});
|
||||
|
||||
test("a provider command payload with bytes after its first value is refused by count, without echoing the tail", () => {
|
||||
const command = { commandId: "c1", commandKind: "k", providerId: "p", serviceInstanceId: "s", payload: { n: 1 } };
|
||||
const whole = documentPayloadBytes(encodeDocumentPayload(providerCommandDocument, command));
|
||||
const tailed = documentPayloadFromBytes(Uint8Array.from([...whole, ...Buffer.from("canary-tail-8c1f")]));
|
||||
assert.throws(
|
||||
() => decodeProviderCommandDocument({
|
||||
schemaId: cultMeshProviderSessionSchemas.command, recordKey: "p/s/c1", storedAt: "2026-10-11T00:00:00Z",
|
||||
payloadEncoding: "messagepack" as const, payload: tailed,
|
||||
}),
|
||||
(error: Error) => error.message.includes("16 bytes after its first MessagePack value") && !error.message.includes("canary-tail-8c1f"),
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
## Unreleased (breaking; version 0.x, so the next minor)
|
||||
|
||||
- `decode_witness_artifact_bundle_payload` (and `CultNetWitnessArtifactBundle.from_payload`) read the payload through `cultcache_py.read_payload_tree`: bytes after its first MessagePack value raise `ValueError` `cultnet.witness_artifact_bundle payload has N bytes after its first MessagePack value (refused on read).` instead of `msgpack.ExtraData`, and a payload nested past 256 raises `CultCachePayloadDepthError`. Needs `cultcache-py` 0.4.0, the release that exports `read_payload_tree`: the dependency is now `cultcache-py>=0.4.0` (it was `>=0.2.0`, which admits releases whose `import cultnet_py` fails), and `tests/test_declared_dependency_ranges.py` fails when a declared bound admits a tagged release lacking a name the package imports. The same check found `cultmesh-py` importing `cache.index_text`, which `cultcache-py` 0.2.0 lacks; its bound is `cultcache-py>=0.4.0` too (it has no changelog of its own). `resolve_document_for_raw_record` reads the schema hint from a raw record's first value whether or not bytes follow it, so a record routed by its stamp is refused by count when applied (`ValueError`, in a snapshot, a shard log and a direct apply) like one routed by a registered schema id; `apply_shard_log_response` used to skip it silently as an unknown schema. A tail on a payload whose first value names no registered schema is still `KeyError`, which a shard log skips.
|
||||
- `apply_raw_document_record` and `apply_raw_snapshot` raise `SchemaConflictError` when a remote record is of another type than the one the cache holds at its key (`cultcache-py` holds one record per key whatever its type); the cache is unchanged, and the caller decides whether to drop the record or the peer. `cultmesh-py`'s server answers such a put with `message_invalid`; its client's `read_next_change` lets the error out.
|
||||
- `CultNetDatabaseChange` carries `commit_id` and `commit_count` (`commitId`/`commitCount`; None outside a multi-record commit, as C# writes nil): `from_wire` refuses an empty or non-string id and a count that is not an integer of at least 2, `to_wire` writes them when set, and the published `database_change_raw` schema lists both. Python applies no batch. The `simulation_fact` document's `shard`, `frame`, `subject` and `claim_kind` aliases are `roles` (many facts share each); the catalog bytes do not move.
|
||||
- Decodes the write receipt frames of `contracts/cultnet/interop/write-receipt.md`: `parse_message` validates `cultnet.write_admitted.v0` and `cultnet.error.v1` against their closed schemas (unknown keys, empty or non-string ids, an unknown operation or code are refused; `error.v1` requires `messageId`, string or null; `limit_reached` is `error.v1` only). Added `CULTNET_ERROR_V1_CODES`, `CULTNET_WRITE_RECEIPT_MESSAGE_VERSIONS`, `write_content_hash` and `hello_advertises_write_receipts`; `hello` takes `supported_message_versions`. `tests/test_write_receipt.py` replays the vectors' `contentHash`, `wire` and `schemaInvalid` sections. Not implemented here: the client write tracker (`scenarios`) and the server rules (`dedupe`, `blankFields`); this package has no receipt server and no write client.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ classifiers = [
|
|||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
]
|
||||
dependencies = ["cultcache-py>=0.4.0", "msgpack>=1.0"]
|
||||
dependencies = ["cultcache-py>=0.2.0", "msgpack>=1.0"]
|
||||
[project.optional-dependencies]
|
||||
crypto = ["cryptography>=42"]
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from dataclasses import dataclass
|
|||
from typing import Any
|
||||
|
||||
import msgpack
|
||||
from cultcache_py import read_payload_tree
|
||||
|
||||
from .unicode_messageid_tables import DEFAULT_IGNORABLE, LETTER_OR_NUMBER
|
||||
|
||||
|
|
@ -545,7 +544,7 @@ def encode_witness_artifact_bundle_payload(bundle: dict[str, Any] | CultNetWitne
|
|||
|
||||
|
||||
def decode_witness_artifact_bundle_payload(payload: bytes) -> dict[str, Any]:
|
||||
slots = read_payload_tree("cultnet.witness_artifact_bundle", payload)
|
||||
slots = msgpack.unpackb(payload, raw=False)
|
||||
if not isinstance(slots, list) or len(slots) < 8:
|
||||
raise ValueError("CultNet witness artifact bundle payload must be an 8-slot MessagePack array")
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -171,14 +171,9 @@ def resolve_document_for_raw_record(
|
|||
|
||||
|
||||
def _infer_schema_version_from_payload(payload: bytes) -> str | None:
|
||||
import msgpack # type: ignore
|
||||
try:
|
||||
import msgpack # type: ignore
|
||||
decoded = msgpack.unpackb(payload, raw=False)
|
||||
except msgpack.ExtraData as extra:
|
||||
# The hint is read from the first value; the tail is not this function's to judge. The routed definition refuses
|
||||
# it by count when the record is applied, as it does for a record routed by a registered schema id. Treating a
|
||||
# tail as "no hint" made a stamp-routed record look like an unknown schema, which a shard log skips silently.
|
||||
decoded = extra.unpacked
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(decoded, list) and decoded and isinstance(decoded[0], str):
|
||||
|
|
|
|||
|
|
@ -1971,49 +1971,6 @@ class CultNetTests(unittest.TestCase):
|
|||
apply_raw_document_record(cache, schema_document_map([first, second]), raw(first)) # the same type still replaces
|
||||
self.assertEqual(cache.get_required(first, "shared").name, "remote")
|
||||
|
||||
def test_a_raw_record_with_a_tail_is_routed_by_its_stamp_and_refused_by_count_on_every_apply_path(self) -> None:
|
||||
import msgpack # type: ignore
|
||||
from cultcache_py import define_document_type
|
||||
from cultnet_py import resolve_document_for_raw_record
|
||||
|
||||
document = define_document_type("tests.tail_note", schema_id="tests.tail_note", schema_version="tests.tail_note.v1",
|
||||
payload_encoder=msgpack.packb, payload_decoder=lambda payload: msgpack.unpackb(payload, raw=False))
|
||||
known = document.catalog_entry().schema_id
|
||||
head = msgpack.packb(["tests.tail_note.v1", "x"], use_bin_type=True)
|
||||
|
||||
def raw(schema_id: str, payload: bytes) -> dict:
|
||||
return {"schemaId": schema_id, "recordKey": "k", "storedAt": "2026-06-13T00:00:00Z",
|
||||
"payloadEncoding": "messagepack", "payload": payload}
|
||||
|
||||
documents = schema_document_map([document])
|
||||
# The stamp is read from the first value whether or not a tail follows, so an unknown schema id whose payload names
|
||||
# a registered schema routes to it; the tail is then refused by count where the record is applied.
|
||||
self.assertIs(resolve_document_for_raw_record(documents, "unknown", raw("unknown", head + b"\xc0")), document)
|
||||
for schema_id in (known, "unknown"):
|
||||
cache = CultCache()
|
||||
cache.register_document_type(document)
|
||||
with self.assertRaises(ValueError, msg=schema_id) as caught:
|
||||
apply_raw_document_record(cache, documents, raw(schema_id, head + b"\xc0"))
|
||||
self.assertIn("1 bytes after its first MessagePack value", str(caught.exception))
|
||||
self.assertIsNone(cache.get(document, "k"))
|
||||
snapshot = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": "s",
|
||||
"documents": [raw(schema_id, head + b"\xc0")]}
|
||||
with self.assertRaises(ValueError, msg=schema_id):
|
||||
apply_raw_snapshot(cache, [document], snapshot)
|
||||
log = {"schemaVersion": "cultnet.shard_log_response.v0", "messageId": "l", "shardId": "s", "shardEpoch": 1,
|
||||
"resyncRequired": False,
|
||||
"entries": [{"sequence": 0, "changeKind": "updated",
|
||||
"put": {"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p",
|
||||
"document": raw(schema_id, head + b"\xc0"), "shardId": "s", "shardEpoch": 1}}]}
|
||||
with self.assertRaises(ValueError, msg=schema_id) as caught:
|
||||
apply_shard_log_response(cache, [document], log)
|
||||
self.assertIn("1 bytes after its first MessagePack value", str(caught.exception))
|
||||
self.assertIsNone(cache.get(document, "k"))
|
||||
# A tail on a payload that names no registered schema is still an unknown schema, which a shard log skips.
|
||||
stranger = msgpack.packb(["tests.stranger.v1", "x"], use_bin_type=True) + b"\xc0"
|
||||
with self.assertRaises(KeyError):
|
||||
resolve_document_for_raw_record(documents, "unknown", raw("unknown", stranger))
|
||||
|
||||
def test_cultnet_replication_helpers_apply_raw_snapshot_and_shard_log(self) -> None:
|
||||
document = define_database_entry_type(
|
||||
"replica.item",
|
||||
|
|
@ -2704,23 +2661,6 @@ class CultNetTests(unittest.TestCase):
|
|||
self.assertEqual(responses[0]["details"]["value"], schema_version)
|
||||
self.assertNotIn("messageId", responses[0])
|
||||
|
||||
def test_witness_artifact_bundle_payload_is_one_value_and_depth_wins_over_a_tail(self) -> None:
|
||||
import msgpack # type: ignore
|
||||
from cultcache_py import CultCachePayloadDepthError
|
||||
|
||||
payload = msgpack.packb(["b", "k", "t", {}, [], [], [], {}], use_bin_type=True)
|
||||
decode_witness_artifact_bundle_payload(payload)
|
||||
for tail, count in ((b"\xc0", 1), (b"\xc1\xc1", 2)):
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
decode_witness_artifact_bundle_payload(payload + tail)
|
||||
self.assertIs(type(caught.exception), ValueError)
|
||||
self.assertEqual(
|
||||
str(caught.exception),
|
||||
f"cultnet.witness_artifact_bundle payload has {count} bytes after its first MessagePack value (refused on read).",
|
||||
)
|
||||
with self.assertRaises(CultCachePayloadDepthError):
|
||||
decode_witness_artifact_bundle_payload(b"\x91" * 300 + b"\xc0\xc0")
|
||||
|
||||
def test_cultnet_simulation_consensus_dedupes_witnesses_and_requires_quorum(self) -> None:
|
||||
from cultnet_py import CultNetSimulationConsensus
|
||||
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
"""A sibling package resolves through the workspace's editable install, so a package that starts importing a new name from
|
||||
cultcache-py passes whatever range it declares. A consumer outside the workspace installs the newest release the range
|
||||
admits. This is the Python form of scripts/test-declared-dependency-ranges.mjs: for every tagged cultcache-py release
|
||||
(`cultcache-py-vX.Y.Z`) that the `cultcache-py>=` bound of cultnet-py or cultmesh-py admits, every name those packages
|
||||
import from cultcache_py must be defined by that release's source. It needs git with the release tags and skips without
|
||||
them. Only `>=` bounds are read; star imports and names reached by attribute access are not checked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import re
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
PACKAGES = ROOT / "packages"
|
||||
CONSUMERS = ("cultnet-py", "cultmesh-py")
|
||||
|
||||
|
||||
def _version(text: str) -> tuple[int, ...]:
|
||||
return tuple(int(part) for part in text.split("."))
|
||||
|
||||
|
||||
def _git(*args: str) -> str | None:
|
||||
try:
|
||||
done = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
return done.stdout
|
||||
|
||||
|
||||
def _lower_bound(package: str) -> tuple[int, ...]:
|
||||
manifest = (PACKAGES / package / "pyproject.toml").read_text(encoding="utf-8")
|
||||
match = re.search(r'"cultcache-py>=(\d+\.\d+\.\d+)"', manifest)
|
||||
assert match, f"{package} declares no cultcache-py>= bound"
|
||||
return _version(match.group(1))
|
||||
|
||||
|
||||
def _imports(package: str) -> dict[str, set[str]]:
|
||||
"""module -> names imported from it, for every `from cultcache_py[.x] import names` under the package's src."""
|
||||
found: dict[str, set[str]] = {}
|
||||
for source in (PACKAGES / package / "src").rglob("*.py"):
|
||||
for node in ast.walk(ast.parse(source.read_text(encoding="utf-8-sig"))):
|
||||
if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
||||
if node.module == "cultcache_py" or node.module.startswith("cultcache_py."):
|
||||
found.setdefault(node.module, set()).update(alias.name for alias in node.names if alias.name != "*")
|
||||
return found
|
||||
|
||||
|
||||
def _defined(tag: str, module: str) -> set[str] | None:
|
||||
base = "packages/cultcache-py/src/" + module.replace(".", "/")
|
||||
for path in (base + ".py", base + "/__init__.py"):
|
||||
text = _git("show", f"{tag}:{path}")
|
||||
if text is None:
|
||||
continue
|
||||
names: set[str] = set()
|
||||
for node in ast.parse(text.lstrip("\ufeff")).body:
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
names.add(node.name)
|
||||
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
names.update((alias.asname or alias.name).split(".")[0] for alias in node.names)
|
||||
elif isinstance(node, (ast.Assign, ast.AnnAssign)):
|
||||
for target in node.targets if isinstance(node, ast.Assign) else [node.target]:
|
||||
if isinstance(target, ast.Name):
|
||||
names.add(target.id)
|
||||
return names
|
||||
return None
|
||||
|
||||
|
||||
class DeclaredDependencyRangeTests(unittest.TestCase):
|
||||
def test_every_admitted_cultcache_release_defines_what_the_consumers_import(self) -> None:
|
||||
listed = _git("tag", "--list", "cultcache-py-v*")
|
||||
tags = sorted((t for t in (listed or "").split() if re.fullmatch(r"cultcache-py-v\d+\.\d+\.\d+", t)),
|
||||
key=lambda t: _version(t.rsplit("v", 1)[1]))
|
||||
if not tags:
|
||||
self.skipTest("git with the cultcache-py release tags is not available")
|
||||
current = re.search(r'^version = "(\d+\.\d+\.\d+)"', (PACKAGES / "cultcache-py" / "pyproject.toml").read_text(encoding="utf-8"), re.M)
|
||||
assert current
|
||||
for package in CONSUMERS:
|
||||
bound = _lower_bound(package)
|
||||
self.assertLessEqual(bound, _version(current.group(1)), f"{package}'s bound is above the workspace cultcache-py")
|
||||
imports = _imports(package)
|
||||
for tag in tags:
|
||||
if _version(tag.rsplit("v", 1)[1]) < bound:
|
||||
continue
|
||||
for module, names in imports.items():
|
||||
defined = _defined(tag, module)
|
||||
self.assertIsNotNone(defined, f"{package} imports {module}, which {tag} does not have")
|
||||
self.assertEqual(sorted(names - defined), [], f"{package} imports names {tag} lacks from {module}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
All notable changes to `@gamecult/cultnet-ts` are recorded here, following
|
||||
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
## [0.4.0]
|
||||
## Unreleased
|
||||
|
||||
### Breaking
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@gamecult/cultnet-ts",
|
||||
"version": "0.4.0",
|
||||
"version": "0.3.0",
|
||||
"description": "CultNet-style typed direct-pipe messaging and state replication for TypeScript runtimes.",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
|
|
|
|||
|
|
@ -392,13 +392,6 @@ test("decodeManifest accepts and refuses exactly what the reference does, and re
|
|||
}
|
||||
});
|
||||
|
||||
test("a manifest with bytes after its first value is refused by count, whatever the tail is", () => {
|
||||
const whole = encodeManifest(manifestOf());
|
||||
for (const tail of [[0xc0], [0xc1, 0xc1], [0x90]]) {
|
||||
assert.throws(() => decodeManifest(Uint8Array.from([...whole, ...tail])), new RegExp(`${tail.length} bytes after its first MessagePack value`));
|
||||
}
|
||||
});
|
||||
|
||||
test("a manifest written as a map names the manifest, not a field inside it", () => {
|
||||
assert.throws(() => decodeManifest(encode({ 0: "a", 9: { k: "v" } })), /field 'manifest' has the wrong type/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ import {
|
|||
SchemaConflictError,
|
||||
SingleFileMessagePackBackingStore,
|
||||
defineDocumentType,
|
||||
documentPayloadBytes,
|
||||
documentPayloadFromBytes,
|
||||
schemaIdentityOf,
|
||||
} from "@gamecult/cultcache-ts";
|
||||
|
||||
|
|
@ -3142,24 +3140,3 @@ test("CultNet registry names the canonical schema id of a compatible alias, and
|
|||
assert.equal(registry.canonicalSchemaId("p3.nobody"), undefined);
|
||||
assert.equal(registry.canonicalSchemaId("p3.shared"), undefined, "an id several bindings claim resolves to none");
|
||||
});
|
||||
|
||||
test("a replicated record with bytes after its first MessagePack value is refused by count and held nowhere", async () => {
|
||||
const definition = defineDocumentType({ type: "tails.note", schemaId: "tails.note.v1", schema: z.object({ body: z.string() }) });
|
||||
const registry = new CultNetDocumentRegistry([defineCultNetDocumentBinding({ definition })]);
|
||||
const origin = CultCache.builder().withDocumentType(definition).build();
|
||||
await origin.put(definition, "k", { body: "x" });
|
||||
const document = registry.createRawSnapshotResponse(origin, "tails").documents[0]!;
|
||||
const whole = documentPayloadBytes(document.payload);
|
||||
const target = CultCache.builder().withDocumentType(definition).build();
|
||||
await assert.rejects(
|
||||
registry.applyRawDocumentPutMessage(target, {
|
||||
schemaVersion: "cultnet.document_put_raw.v0",
|
||||
messageId: "tails",
|
||||
document: { ...document, payload: documentPayloadFromBytes(Uint8Array.from([...whole, 0xc0, 0xc1])) },
|
||||
}),
|
||||
(error: Error) => error.message.includes('document type "tails.note" payload has 2 bytes after its first MessagePack value'),
|
||||
);
|
||||
assert.deepEqual(target.snapshot(), []);
|
||||
await registry.applyRawDocumentPutMessage(target, { schemaVersion: "cultnet.document_put_raw.v0", messageId: "whole", document });
|
||||
assert.deepEqual(target.getRequired(definition, "k"), { body: "x" });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -857,7 +857,7 @@ test("workflow: each job releases only through release-after-suite.mjs", () => {
|
|||
});
|
||||
|
||||
test("release-after-suite checks the tag, then runs the suite, the semver check and the package's tests, and releases only when all passed", async () => {
|
||||
const { releaseAfterSuite, suite, checker, rangeCheck } = await import("./release-after-suite.mjs");
|
||||
const { releaseAfterSuite, suite, checker } = await import("./release-after-suite.mjs");
|
||||
const npmVersion = JSON.parse(readFileSync(join(repoRoot, "packages", "cultcache-ts", "package.json"), "utf8")).version;
|
||||
const npmDir = join(repoRoot, "packages", "cultcache-ts");
|
||||
const pyDir = join(repoRoot, "packages", "cultnet-py");
|
||||
|
|
@ -881,15 +881,13 @@ test("release-after-suite checks the tag, then runs the suite, the semver check
|
|||
const npmCalls = [
|
||||
suiteCall,
|
||||
checkerCall("@gamecult/cultcache-ts", npmVersion),
|
||||
["npm", ["run", "build:ts"], repoRoot, checked],
|
||||
[process.execPath, [rangeCheck], repoRoot, checked],
|
||||
["npm", ["test"], npmDir, checked],
|
||||
["npm", ["publish", "--access", "public"], npmDir, env],
|
||||
];
|
||||
assert.equal(go(["npm-publish", npmDir, npmTag]), 0);
|
||||
assert.deepEqual(calls, npmCalls);
|
||||
// any check that fails stops the release at that check
|
||||
for (const fail of [0, 1, 2, 3, 4]) {
|
||||
for (const fail of [0, 1, 2]) {
|
||||
assert.equal(go(["npm-publish", npmDir, npmTag], fail), 1, `check ${fail}`);
|
||||
assert.deepEqual(calls, npmCalls.slice(0, fail + 1), `check ${fail}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
// How publish-packages.yml releases a package. From this one process, in order: the release tag must name
|
||||
// the package's declared prefix and its manifest version, the release-checks suite must pass, the semver
|
||||
// policy check must pass for that version, an npm release's declared dependency ranges must hold (the
|
||||
// workspace is built, then every tagged release a sibling's range admits is checked against what the
|
||||
// sibling imports), the package's own tests must pass, and only then the release action runs. No workflow
|
||||
// condition can then release what any of those checks did not pass.
|
||||
// policy check must pass for that version, the package's own tests must pass, and only then the release
|
||||
// action runs. No workflow condition can then release what any of those checks did not pass.
|
||||
// docs/semver-policy.md, "The suite gates every release".
|
||||
//
|
||||
// node scripts/release-after-suite.mjs npm-publish <package dir> <git ref> npm publish --access public in <dir>
|
||||
|
|
@ -20,7 +18,6 @@ import releasePackages from "./release-packages.mjs";
|
|||
|
||||
export const suite = fileURLToPath(new URL("./check-changelog-semver.test.mjs", import.meta.url));
|
||||
export const checker = fileURLToPath(new URL("./check-changelog-semver.mjs", import.meta.url));
|
||||
export const rangeCheck = fileURLToPath(new URL("./test-declared-dependency-ranges.mjs", import.meta.url));
|
||||
const repoRoot = fileURLToPath(new URL("..", import.meta.url));
|
||||
|
||||
// The checks run without the variables that can make `node --test` run nothing and exit 0
|
||||
|
|
@ -37,8 +34,6 @@ export const actions = {
|
|||
"npm-publish": {
|
||||
manifest: (dir) => JSON.parse(readFileSync(join(dir, "package.json"), "utf8")),
|
||||
test: (dir) => ["npm", ["test"], dir],
|
||||
// The range check builds each tagged release against the workspace's built dists, so it follows a workspace build.
|
||||
ranges: [["npm", ["run", "build:ts"], repoRoot], [process.execPath, [rangeCheck], repoRoot]],
|
||||
release: (dir) => ["npm", ["publish", "--access", "public"], dir],
|
||||
tagOnly: true,
|
||||
},
|
||||
|
|
@ -88,9 +83,6 @@ export function releaseAfterSuite(argv, run = spawn, env = process.env) {
|
|||
if (isTag && run(process.execPath, [checker, "--package", manifest.name, "--version", manifest.version, "--cwd", repoRoot], undefined, clean) !== 0) {
|
||||
return refuse(`${manifest.name} ${manifest.version} fails the semver policy (docs/semver-policy.md)`);
|
||||
}
|
||||
for (const [command, args, cwd] of isTag ? action.ranges ?? [] : []) {
|
||||
if (run(command, args, cwd, clean) !== 0) return refuse(`${manifest.name}'s declared dependency ranges do not hold (scripts/test-declared-dependency-ranges.mjs)`);
|
||||
}
|
||||
if (run(...action.test(dir), clean) !== 0) return refuse(`${manifest.name}'s own tests failed`);
|
||||
return run(...action.release(dir), env);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import { fileURLToPath } from "node:url";
|
|||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const rootRequire = createRequire(join(repoRoot, "package.json"));
|
||||
const ts = rootRequire("typescript");
|
||||
const npmCli = join(npmRoot(), "bin", "npm-cli.js");
|
||||
const semver = createRequire(join(npmRoot(), "package.json"))("semver");
|
||||
|
||||
// npm ships semver, so the check needs no dependency of its own. npm's root is beside the node binary on Windows and under lib/ on Unix.
|
||||
|
|
@ -85,17 +84,16 @@ function sourceFiles(directory) {
|
|||
});
|
||||
}
|
||||
|
||||
// Builds the package as the tag released it (with the repo-level contracts its generator reads), in a scratch directory that borrows the workspace's node_modules.
|
||||
// Builds the package as the tag released it, in a scratch directory that borrows the workspace's node_modules.
|
||||
function buildRelease(directory, tag, scratch) {
|
||||
const archive = join(scratch, `${tag}.tar`);
|
||||
git("archive", "--format=tar", "-o", archive, tag, `packages/${directory}`, "contracts");
|
||||
git("archive", "--format=tar", "-o", archive, tag, `packages/${directory}`);
|
||||
const extracted = join(scratch, tag);
|
||||
mkdirSync(extracted);
|
||||
execFileSync("tar", ["-xf", archive, "-C", extracted]);
|
||||
symlinkSync(join(repoRoot, "node_modules"), join(extracted, "node_modules"), "junction");
|
||||
const packageRoot = join(extracted, "packages", directory);
|
||||
// The package's own build, which is how a release was made: cultnet-ts generates its contracts module first.
|
||||
execFileSync(process.execPath, [npmCli, "run", "build", "--silent"], { cwd: packageRoot, stdio: "inherit" });
|
||||
execFileSync(process.execPath, [rootRequire.resolve("typescript/bin/tsc"), "-p", "tsconfig.json"], { cwd: packageRoot, stdio: "inherit" });
|
||||
return rootRequire(join(packageRoot, "dist", "index.js"));
|
||||
}
|
||||
|
||||
|
|
@ -129,7 +127,7 @@ try {
|
|||
const prefix = `${dependency.directory}-v`;
|
||||
const admitted = tags.filter((tag) => {
|
||||
const version = tag.startsWith(prefix) ? tag.slice(prefix.length) : undefined;
|
||||
return version !== undefined && semver.valid(version) !== null && semver.satisfies(version, range);
|
||||
return version !== undefined && semver.valid(version) !== null && semver.satisfies(version, range) && version !== dependency.manifest.version;
|
||||
});
|
||||
for (const tag of admitted) {
|
||||
if (!built.has(tag)) {
|
||||
|
|
|
|||
|
|
@ -1,71 +0,0 @@
|
|||
// test-declared-dependency-ranges.mjs against a fixture repo: a dependency that gained an export a dependent imports,
|
||||
// while its manifest still carries the version of its latest release tag, must fail the check, because a consumer's
|
||||
// range admits that tagged release and it lacks the export. Bumping the version and the range makes it pass.
|
||||
// The fixture package a builds like cultnet-ts: its build script first generates a module from the repo-level contracts/
|
||||
// directory, so a tag built by a bare tsc, or archived without contracts/, fails here as it does on the real tree.
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const script = join(repoRoot, "scripts", "test-declared-dependency-ranges.mjs");
|
||||
|
||||
const write = (root, path, text) => {
|
||||
mkdirSync(dirname(join(root, path)), { recursive: true });
|
||||
writeFileSync(join(root, path), text);
|
||||
};
|
||||
const git = (root, ...args) => execFileSync("git", ["-c", "user.name=t", "-c", "user.email=t@t", ...args], { cwd: root, stdio: "pipe" });
|
||||
const tsc = "node ../../node_modules/typescript/bin/tsc -p tsconfig.json";
|
||||
const manifest = (name, version, extra = {}, build = tsc) =>
|
||||
JSON.stringify({ name: `@t/${name}`, version, scripts: { build }, ...extra });
|
||||
const generated = "node tools/generate.mjs && " + tsc;
|
||||
const tsconfig = JSON.stringify({ compilerOptions: { outDir: "dist", module: "commonjs", target: "es2020", declaration: false }, include: ["src"] });
|
||||
|
||||
function fixture() {
|
||||
const root = mkdtempSync(join(tmpdir(), "declared-ranges-fixture-"));
|
||||
write(root, "package.json", JSON.stringify({ name: "fixture", private: true, workspaces: ["packages/a", "packages/b"] }));
|
||||
write(root, "contracts/value.json", "2");
|
||||
write(root, "packages/a/tools/generate.mjs",
|
||||
'import { readFileSync, writeFileSync } from "node:fs";\n' +
|
||||
'writeFileSync("src/data.ts", `export const contractValue = ${readFileSync("../../contracts/value.json", "utf8")};\\n`);\n');
|
||||
write(root, "packages/a/package.json", manifest("a", "1.0.0", {}, generated));
|
||||
write(root, "packages/a/tsconfig.json", tsconfig);
|
||||
write(root, "packages/a/src/index.ts", 'export { contractValue } from "./data";\nexport const foo = 1;\n');
|
||||
write(root, "packages/b/package.json", manifest("b", "1.0.0", { dependencies: { "@t/a": "^1.0.0" } }));
|
||||
write(root, "packages/b/tsconfig.json", tsconfig);
|
||||
write(root, "packages/b/src/index.ts", 'import { foo } from "@t/a";\nexport const b = foo;\n');
|
||||
mkdirSync(join(root, "scripts"));
|
||||
copyFileSync(script, join(root, "scripts", "test-declared-dependency-ranges.mjs"));
|
||||
symlinkSync(join(repoRoot, "node_modules"), join(root, "node_modules"), "junction");
|
||||
git(root, "init", "-q", "-b", "main");
|
||||
git(root, "add", "package.json", "contracts", "packages", "scripts");
|
||||
git(root, "commit", "-q", "-m", "release a 1.0.0");
|
||||
git(root, "tag", "a-v1.0.0");
|
||||
return root;
|
||||
}
|
||||
|
||||
const check = (root) => spawnSync(process.execPath, [join(root, "scripts", "test-declared-dependency-ranges.mjs")], { cwd: root, encoding: "utf8" });
|
||||
|
||||
test("a dependency that gained an export a dependent imports, still at its latest tag's version, fails the check", () => {
|
||||
const root = fixture();
|
||||
try {
|
||||
assert.equal(check(root).status, 0, "the fixture passes before the export is added");
|
||||
|
||||
write(root, "packages/a/src/index.ts", 'export { contractValue } from "./data";\nexport const foo = 1;\nexport const bar = 2;\n');
|
||||
write(root, "packages/b/src/index.ts", 'import { foo, bar } from "@t/a";\nexport const b = foo + bar;\n');
|
||||
const stale = check(root);
|
||||
assert.notEqual(stale.status, 0, stale.stdout);
|
||||
assert.match(stale.stderr, /a-v1\.0\.0 does not export bar/u);
|
||||
|
||||
write(root, "packages/a/package.json", manifest("a", "1.1.0", {}, generated));
|
||||
write(root, "packages/b/package.json", manifest("b", "1.0.0", { dependencies: { "@t/a": "^1.1.0" } }));
|
||||
const bumped = check(root);
|
||||
assert.equal(bumped.status, 0, bumped.stderr);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
<ProjectReference Include="..\GameCult.Caching\GameCult.Caching.csproj" />
|
||||
<ProjectReference Include="..\..\packages\cultmath\src\CultMath\CultMath.csproj" />
|
||||
<PackageReference Include="MessagePack" Version="3.1.7" />
|
||||
<PackageReference Include="System.Text.Json" Version="8.0.5" />
|
||||
<PackageReference Include="System.Text.Json" Version="9.0.10" />
|
||||
<PackageReference Include="MessagePackAnalyzer" Version="3.1.7" PrivateAssets="all" />
|
||||
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
|
||||
<InternalsVisibleTo Include="GameCult.Caching.GeneratorConsumer" />
|
||||
|
|
|
|||
|
|
@ -315,138 +315,11 @@ refuses to replace a file whose header it cannot read, and leaves it as it was.
|
|||
in that store, and leaves the store clean.
|
||||
- A conditional commit lands the batch onto the file as it is under the lock.
|
||||
|
||||
C#, Rust, TypeScript and Python single-file writers exclude each other on the
|
||||
shared lock file. Another runtime need not obey the `storedAt` rule.
|
||||
|
||||
## Operating a single-file store
|
||||
|
||||
What the lock guarantees, per runtime. Every claim cites the code.
|
||||
|
||||
- **Scope.** The lock is the sidecar `<path>.lock`, taken by every write of a
|
||||
single-file store, never by a read except in Rust. It is advisory: it excludes
|
||||
only openers that also take it, in this process and others. C# opens it
|
||||
share-none (`CultCache.cs:5166-5183`; .NET emulates that with `flock(LOCK_EX)`
|
||||
on Unix, switched off by `DOTNET_SYSTEM_IO_DISABLEFILELOCKING`, see
|
||||
`docs/research/cross-process-file-locking-prior-art.md`); Rust takes
|
||||
`fs2` exclusive (`cultcache-rs/src/lib.rs:1721-1726`); TypeScript `flock` or
|
||||
`LockFileEx` (`cultcache-ts/src/store-file.ts:340-382`); Python the same
|
||||
(`cultcache-py/src/cultcache_py/stores.py:342-389`). A tool that opens and
|
||||
writes the `.cc` file without the lock is not excluded. `flock` is not
|
||||
reliable on every network filesystem; keep stores on a local one.
|
||||
- **What a second writer sees.** It waits. C# polls every 10 ms for 30 seconds,
|
||||
then the `IOException` propagates (`CultCache.cs:5166-5183`); with `wait: false`
|
||||
(`TryCommit`) it returns `Contended` (`CultCache.cs:2052`, `2897`, `4691-4693`).
|
||||
TypeScript times out after 30 s (`store-file.ts:173-195`), Python raises
|
||||
`TimeoutError` after `lock_timeout`, 30 s by default (`stores.py:342-354`).
|
||||
Rust waits without a timeout on its write paths (`lib.rs:1721-1726`); only its
|
||||
`try_with_exclusive_lock` returns without waiting (`lib.rs:1832-1843`). Each
|
||||
write re-reads the file after taking the lock and applies its staged keys onto
|
||||
it (`CultCache.cs:4659-4676`), so waiting writers merge rather than overwrite.
|
||||
The in-memory cache of a writer is not refreshed by another's write until its
|
||||
next `PullAll`, which does nothing while that cache is dirty
|
||||
(`CultCache.cs:4582-4588`).
|
||||
- **What a reader sees.** C#, TypeScript and Python read without the lock
|
||||
(`CultCache.cs:5139`, `5227`; `single-file-messagepack-backing-store.ts:200`;
|
||||
`stores.py:144-152`). The store file is only ever replaced by rename of a fully
|
||||
written, fsynced temp file (`CultCache.cs:5191-5213`, `lib.rs:1693-1712`,
|
||||
`store-file.ts:70-95`, `stores.py:316-326`), so a read gets one complete
|
||||
generation, old or new, never a mix. Rust `pull_all` takes the lock shared and
|
||||
so waits behind a writer (`lib.rs:1926-1930`); it creates the `.lock` file if
|
||||
absent, so on a read-only directory use `pull_all_read_only_snapshot`
|
||||
(`lib.rs:1667-1669`), which takes no lock and relies on the producer's atomic
|
||||
replace.
|
||||
- **Not covered.** A unique-index check or condition spanning two stores, or any
|
||||
invariant between two files: each store has its own lock and no store carries a
|
||||
generation counter (the file is `[formatVersion, schemaCatalog, records]`);
|
||||
a record's identity is its `version`, the SHA-256 of its stored bytes. No
|
||||
library writer holds two store locks at once (`CultCache.cs:2917-2926` judges
|
||||
one store, releases, then commits the home). Python's `JsonLinesBackingStore`
|
||||
takes only an in-process lock (`stores.py:73-80`) and is not a single-file store
|
||||
in this sense.
|
||||
- **The `.lock` file is not a stale-lock marker.** It is never written or deleted
|
||||
by a library writer (`store-file.ts:167-172`, `stores.py:342-345`), and the
|
||||
lock is released when the holding process exits. A `.lock` file left on disk is
|
||||
normal. Do not delete it while any process may open the store: a new opener
|
||||
would create a second file and take a second, independent lock on it.
|
||||
|
||||
### A leftover temp file
|
||||
|
||||
A write stages its bytes in a temp file beside the store and renames it over the
|
||||
store. The name is consumed by the rename, so a temp file that still exists is a
|
||||
write that was never applied. The store file itself is never partial because of
|
||||
it.
|
||||
|
||||
| Runtime | Temp name | Source |
|
||||
|---|---|---|
|
||||
| C# | `.<store file name>.<32 hex>.tmp` | `CultCache.cs:5191` |
|
||||
| Rust | `<store file name>.<uuid>.tmp` | `lib.rs:4268-4275` |
|
||||
| TypeScript | `<path>.tmp-<pid>-<ms>-<random>` | `store-file.ts:70` |
|
||||
| Python | `<path>.tmp-<pid>-<uuid hex>` | `stores.py:316` |
|
||||
|
||||
- **Which temp files are safe to delete.** One that no writer is writing:
|
||||
delete only while you hold `<path>.lock` yourself
|
||||
(`flock <path>.lock rm <temp file>`, Linux, util-linux) or with every writer of
|
||||
the store stopped. Merely testing that the lock is free is not enough: a writer
|
||||
can take it and stage a new temp file the moment after the test. While you hold
|
||||
the lock, every temp file of that store is abandoned. Deleting one a writer is
|
||||
still writing destroys that write.
|
||||
- **How to know the write was incomplete.** The temp file existing is the proof.
|
||||
Its content may be partial (killed during the write) or whole (killed between
|
||||
the fsync and the rename); either way it was not applied, and the writer that
|
||||
made it never saw its flush succeed. The store holds the last completed write.
|
||||
Do not rename a temp file over the store by hand. Rust additionally reports a
|
||||
rename that landed but whose directory fsync failed as `ReplacedNotDurable`
|
||||
(`lib.rs:1685-1688`; TypeScript `StoreReplacedNotDurableError`,
|
||||
`store-file.ts:88-92`): after a power loss the store may still be the earlier
|
||||
generation.
|
||||
- **What the store does about it.** Opening and reading ignore temp files and
|
||||
never delete them. A writer removes only its own temp file when its write
|
||||
throws (`CultCache.cs:5216-5222`, `store-file.ts:82-84`, `stores.py:327-331`,
|
||||
`lib.rs:1707-1710`); a killed process skips that. Rust alone sweeps: its next
|
||||
write, under the exclusive lock, deletes every regular file named
|
||||
`<store file name>.<uuid>.tmp` (`lib.rs:1698`, `4318-4352`). C#, TypeScript and
|
||||
Python never sweep, and their unique names mean leftovers accumulate until an
|
||||
operator deletes them. A store file that cannot be read (truncated, wrong
|
||||
header) is refused at open and left byte-identical (the open reads through
|
||||
`ReadSnapshot`, `CultCache.cs:4590`, `5104-5118`, whose `ReadStore` must throw
|
||||
`CultStoreUnreadableException` for anything that is not exactly one store,
|
||||
`4510-4516`); that is the store, not a temp file, and is
|
||||
restored from a copy, never deleted and rewritten.
|
||||
|
||||
### Backing up a pair of stores
|
||||
|
||||
Each store is whole on its own: copying one `.cc` file is safe at any moment,
|
||||
because the file is only ever replaced whole, and a copy of the path is one
|
||||
complete generation. Two stores have two independent locks and no shared
|
||||
generation, so copying them one after the other can capture store A at one
|
||||
moment and store B at a later one. If an application invariant spans the pair,
|
||||
that copy may violate it. Back up the pair at one point in time:
|
||||
|
||||
1. **Preferred, quiesce.** Stop (or have flush and close) every process that
|
||||
writes either store. Flush staged writes first; `PullAll` and the dispose
|
||||
flush do not capture unflushed state of a still-running cache. Copy both `.cc`
|
||||
files. Do not copy `.lock` files or temp files. Start the writers again.
|
||||
2. **Without stopping (Unix): per-store consistency only.** Take both locks in
|
||||
one fixed order (sorted by full path), copy while holding both, release:
|
||||
`flock A.cc.lock flock B.cc.lock cp A.cc B.cc /backup/dir/`.
|
||||
This does not make the pair consistent. A writer holds one store lock at a
|
||||
time, so it can land A and then B with a gap between; holding both locks
|
||||
cannot pause a writer that has landed A and not yet B, so the copy can hold A
|
||||
after and B before that pair of writes. It guarantees each copy is one
|
||||
complete generation of its store and nothing more. Any invariant spanning the
|
||||
pair requires step 1. Cooperating writers wait (30 s in C#, TypeScript and
|
||||
Python; Rust waits indefinitely), so keep the hold short. Unflushed state of
|
||||
live caches is not in it. On Windows, no shipped tool holds the C# share-none
|
||||
lock for you: quiesce.
|
||||
3. **Verify.** Open each copy read-only with the runtime that will use it (for
|
||||
Rust, `pull_all_read_only_snapshot`); an unreadable copy is refused, not
|
||||
repaired (`CultCache.cs:4510-4516`, `5104-5118`).
|
||||
4. **Restore both files together** from the same backup, with every writer
|
||||
stopped, then restart so each cache loads fresh. Copy each file to a temp
|
||||
name in the store's own directory and `mv` it over the store: a plain `cp`
|
||||
over the store is not atomic, and C#, TypeScript and Python readers take no
|
||||
lock, so they can read a torn file. Restoring one half of a pair from a
|
||||
different backup recreates the skew. Leave the `.lock` files as they are.
|
||||
C#, Rust and TypeScript writers exclude each other on the shared lock file;
|
||||
Python takes no cross-process lock yet (`variants:follow_up:py-shared-write-lock`),
|
||||
and Rust's open fails rather than waits while a C# writer holds the lock
|
||||
(`variants:follow_up:rust-lock-open-waits`). Another runtime need not obey the
|
||||
`storedAt` rule.
|
||||
|
||||
## Serialization options
|
||||
|
||||
|
|
|
|||
|
|
@ -307,19 +307,8 @@ Useful surfaces:
|
|||
This is built for callers like Aquarium that want durable runtime state while
|
||||
still choosing *when* to pay the write cost.
|
||||
|
||||
A single-file store admits any number of writers, in one process or many, one at
|
||||
a time. Every write (flush, `PushAll`, `CommitBatch`, cacheless `WriteRecord`)
|
||||
takes an exclusive advisory lock on the sidecar `<path>.lock`, reads the file as
|
||||
it is then, applies its own staged keys onto it and replaces the file by a
|
||||
write-to-temp-then-rename. Two writers never interleave bytes, and a record
|
||||
another writer put there survives; a key both staged holds the last write
|
||||
(`Contracts/cultcache-store-composition.md`, "Locking"). What the lock does not
|
||||
give is cross-key or cross-store atomicity, a lock that binds a program that
|
||||
ignores it, or protection on a filesystem without working `flock`. A second
|
||||
writer waits (30 seconds, then `IOException`; `TryCommit` reports `Contended`
|
||||
instead of waiting); readers take no lock and always see one complete file. The
|
||||
operator procedures for a leftover temp file and for backing up two stores are
|
||||
in the same contract document, "Operating a single-file store".
|
||||
Single-file MessagePack remains a single-writer format. The atomic replace path
|
||||
protects against partial writes, not competing writers trampling each other.
|
||||
|
||||
## Schema migration diagnostics
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
<ProjectReference Include="..\GameCult.Caching\GameCult.Caching.csproj" />
|
||||
<ProjectReference Include="..\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj" />
|
||||
<PackageReference Include="MessagePack" Version="3.1.7" />
|
||||
<PackageReference Include="System.Text.Json" Version="8.0.5" />
|
||||
<PackageReference Include="System.Text.Json" Version="9.0.10" />
|
||||
</ItemGroup>
|
||||
|
||||
<Import Project="..\GameCult.Caching.MessagePack\buildTransitive\GameCult.Caching.MessagePack.targets" />
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
<ProjectReference Include="..\GameCult.Caching.MessagePack\GameCult.Caching.MessagePack.csproj" />
|
||||
<PackageReference Include="MessagePack" Version="3.1.7" />
|
||||
<PackageReference Include="MessagePackAnalyzer" Version="3.1.7" PrivateAssets="all" />
|
||||
<PackageReference Include="System.Text.Json" Version="8.0.5" />
|
||||
<PackageReference Include="System.Text.Json" Version="9.0.10" />
|
||||
<PackageReference Include="R3" Version="1.3.0" />
|
||||
<PackageReference Include="LiteNetLib" Version="2.1.2" />
|
||||
<PackageReference Include="Isopoh.Cryptography.Argon2" Version="2.0.0" />
|
||||
|
|
|
|||
|
|
@ -51,12 +51,6 @@ static async Task<int> ProgramMainAsync(string[] args)
|
|||
case "edit-in-place":
|
||||
await EditInPlaceAsync(file, RequireArg(options, "kind"));
|
||||
return 0;
|
||||
case "write-shapes":
|
||||
await WriteShapesAsync(file);
|
||||
return 0;
|
||||
case "read-shapes":
|
||||
await ReadShapesAsync(file);
|
||||
return 0;
|
||||
case "hold-lock":
|
||||
HoldLock(file, int.Parse(RequireArg(options, "ms"), CultureInfo.InvariantCulture));
|
||||
return 0;
|
||||
|
|
@ -102,49 +96,6 @@ static async Task WriteAsync(string file, string runtimeId)
|
|||
WriteJsonLine(note);
|
||||
}
|
||||
|
||||
// Records a reader of a Bifrost-shaped store must open: a union field (two arms) and nested bytes (a bin member, a bin list).
|
||||
static async Task WriteShapesAsync(string file)
|
||||
{
|
||||
var cache = BuildCache(file);
|
||||
await cache.PullAllBackingStoresAsync();
|
||||
foreach (var shape in InteropShapes())
|
||||
await cache.AddAsync(shape, new CultRecordHandle<CultCacheInteropShape>(new CultRecordKey(shape.DocumentId)));
|
||||
cache.FlushAllBackingStores();
|
||||
WriteShapesJsonLine(InteropShapes());
|
||||
}
|
||||
|
||||
static async Task ReadShapesAsync(string file)
|
||||
{
|
||||
var cache = BuildCache(file);
|
||||
await cache.PullAllBackingStoresAsync();
|
||||
WriteShapesJsonLine(cache.AllEntries.OfType<CultCacheInteropShape>());
|
||||
}
|
||||
|
||||
static CultCacheInteropShape[] InteropShapes() =>
|
||||
[
|
||||
new() { DocumentId = "shape:text", Arm = new InteropTextArm { Text = "union arm 0" }, Raw = Enumerable.Range(0, 256).Select(i => (byte)i).ToArray() },
|
||||
new()
|
||||
{
|
||||
DocumentId = "shape:pack",
|
||||
Arm = new InteropPackArm { Label = "union arm 1", Chunks = [new byte[] { 0xc0, 0xc1, 0x00 }, [], [0x92, 0x01, 0x02]] },
|
||||
Raw = [0xde, 0xad, 0xbe, 0xef]
|
||||
}
|
||||
];
|
||||
|
||||
// One line, ordered by document id, in the form the Rust peer prints.
|
||||
static void WriteShapesJsonLine(IEnumerable<CultCacheInteropShape> shapes) =>
|
||||
Console.Out.WriteLine(JsonSerializer.Serialize(shapes.OrderBy(shape => shape.DocumentId, StringComparer.Ordinal).Select(shape => new
|
||||
{
|
||||
documentId = shape.DocumentId,
|
||||
arm = shape.Arm switch
|
||||
{
|
||||
InteropTextArm text => (object)new { kind = "text", text = text.Text },
|
||||
InteropPackArm pack => new { kind = "pack", label = pack.Label, chunks = pack.Chunks.Select(Convert.ToHexString).ToArray() },
|
||||
_ => throw new InvalidOperationException("Unknown arm.")
|
||||
},
|
||||
raw = Convert.ToHexString(shape.Raw)
|
||||
})));
|
||||
|
||||
static async Task ReadAsync(string file)
|
||||
{
|
||||
var cache = BuildCache(file);
|
||||
|
|
@ -377,28 +328,3 @@ public sealed class CultCacheInteropMark
|
|||
[Key(0)] public string Label { get; set; } = string.Empty;
|
||||
[Key(1)] [CultElementId] public string Id { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
[CultDocument("cultcache.interop-shape", "cultcache.interop_shape.v1")]
|
||||
[MessagePackObject]
|
||||
public sealed class CultCacheInteropShape
|
||||
{
|
||||
[Key(0)] [CultName] public string DocumentId { get; set; } = string.Empty;
|
||||
[Key(1)] public InteropArm Arm { get; set; } = new InteropTextArm();
|
||||
[Key(2)] public byte[] Raw { get; set; } = Array.Empty<byte>();
|
||||
}
|
||||
|
||||
[Union(0, typeof(InteropTextArm)), Union(1, typeof(InteropPackArm))]
|
||||
public abstract class InteropArm { }
|
||||
|
||||
[MessagePackObject]
|
||||
public sealed class InteropTextArm : InteropArm
|
||||
{
|
||||
[Key(0)] public string Text { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
[MessagePackObject]
|
||||
public sealed class InteropPackArm : InteropArm
|
||||
{
|
||||
[Key(0)] public string Label { get; set; } = string.Empty;
|
||||
[Key(1)] public List<byte[]> Chunks { get; set; } = new();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,15 +110,6 @@ namespace GameCult.Caching.Tests
|
|||
Assert.That(message, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"1 {Clause}").And.Contain("refused on read"));
|
||||
}
|
||||
|
||||
// The inspector reads documents through CultCodec.Deserialize (type == document), not DecodeDocument: its branch is its own path.
|
||||
[Test]
|
||||
public void CodecDeserializeOfADocumentRefusesATail()
|
||||
{
|
||||
var codec = CultCacheMessagePack.CreateCodec(Registry);
|
||||
var message = Message(() => codec.Deserialize(typeof(UnionSurface), typeof(UnionSurface), Base.Concat(new byte[] { 0xc0 }).ToArray()));
|
||||
Assert.That(message, Does.Contain(typeof(UnionSurface).FullName).And.Contain($"1 {Clause}").And.Contain("refused on read"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void StoreLoadRefuses()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -58,22 +58,4 @@ public class BytesAfterPathTests
|
|||
.MakeGenericMethod(typeof(CultMeshCdnArtifactManifest));
|
||||
Refusal(() => read.Invoke(null, new object[] { snapshot, record.SchemaId, record.RecordKey })).Should().Contain(Clause);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASingleFileDocumentReadRefusesATail()
|
||||
{
|
||||
var path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"cultlib-bytesafter-mesh-{Guid.NewGuid():N}.cc");
|
||||
try
|
||||
{
|
||||
var descriptor = CultDocumentRegistry.Shared.GetRequired<CultMeshCdnArtifactManifest>();
|
||||
var schema = new CultMeshSingleFileDocumentSchema(descriptor.SchemaId, descriptor.SchemaName, descriptor.SchemaVersion);
|
||||
var payload = WithTail(Manifest()).Payload;
|
||||
CultMesh.WriteSingleFileDocumentPayload(path, new CultRecordKey("k"), schema, null, payload);
|
||||
Refusal(() => CultMesh.ReadSingleFileDocument<CultMeshCdnArtifactManifest>(path, new CultRecordKey("k"))).Should().Contain(Clause);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.IO.File.Delete(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,15 +21,7 @@ namespace GameCult.Networking.Tests
|
|||
[Key(1)] public int N = 3;
|
||||
}
|
||||
|
||||
[CultDocument("tests.bytes_after_deep", "tests.bytes_after_deep.v1")]
|
||||
[MessagePackObject]
|
||||
public sealed class DeepDoc
|
||||
{
|
||||
[Key(0)] [CultName] public string Id = "a";
|
||||
[Key(1)] public DeepDoc Inner;
|
||||
}
|
||||
|
||||
private static readonly CultDocumentRegistry Documents = CultDocumentRegistry.ForTypes(new[] { typeof(TailDoc), typeof(DeepDoc) });
|
||||
private static readonly CultDocumentRegistry Documents = CultDocumentRegistry.ForTypes(new[] { typeof(TailDoc) });
|
||||
|
||||
private static byte[] WithTail() =>
|
||||
CultDocumentMessagePackSerialization.SerializeUntyped(new TailDoc(), typeof(TailDoc), Documents).Concat(new byte[] { 0xc0 }).ToArray();
|
||||
|
|
@ -66,28 +58,5 @@ namespace GameCult.Networking.Tests
|
|||
var registry = new CultNetDocumentRegistry(Documents);
|
||||
Refuses(() => registry.DeserializeRawDocument(Raw(registry, WithTail())));
|
||||
}
|
||||
|
||||
// The write side of the same rule: a record with no binding is serialized by SerializeUntyped, and a payload nested past
|
||||
// the limit is refused there, before it can reach the wire.
|
||||
[Test]
|
||||
public void ToRawRecordWithNoBindingRefusesAPayloadNestedPastTheLimit()
|
||||
{
|
||||
var registry = new CultNetDocumentRegistry(Documents);
|
||||
var descriptor = Documents.GetRequired(typeof(DeepDoc));
|
||||
DeepDoc Chain(int links)
|
||||
{
|
||||
var head = new DeepDoc();
|
||||
var tip = head;
|
||||
for (var i = 0; i < links; i++) tip = tip.Inner = new DeepDoc();
|
||||
return head;
|
||||
}
|
||||
|
||||
// Each link is one array; the root is depth 1, so 255 links is depth 256 (at the limit) and 256 links is 257.
|
||||
Assert.That(registry.ToRawRecord(descriptor, new CultRecordKey("a"), Chain(255), "1970-01-01T00:00:00.000Z").Payload, Is.Not.Empty);
|
||||
var refused = Assert.Throws<MessagePackSerializationException>(
|
||||
() => registry.ToRawRecord(descriptor, new CultRecordKey("a"), Chain(256), "1970-01-01T00:00:00.000Z"))!;
|
||||
Assert.That(refused.Message, Does.Contain("refused on write"));
|
||||
Assert.That(refused.Message, Does.Contain(typeof(DeepDoc).FullName));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue