716 lines
34 KiB
Python
716 lines
34 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterator, TypeVar
|
|
|
|
import msgpack
|
|
|
|
from .documents import UnreadSlots
|
|
from .backing_store import (
|
|
CultCacheEnvelope,
|
|
CultCacheSchemaCatalogEntry,
|
|
CultCacheSchemaCatalogMember,
|
|
)
|
|
|
|
STORE_FORMAT_VERSION = "cultcache.store.v1"
|
|
# A store that can hold element ids. A reader older than element ids refuses this header: it would skip the id slots of a
|
|
# nested element and rewrite the element without them. This runtime writes it back only for a store it read under it; it
|
|
# does not decide when a document carries ids.
|
|
STORE_FORMAT_ELEMENT_IDS = "cultcache.store.v3"
|
|
_READABLE_STORE_FORMATS = (STORE_FORMAT_VERSION, STORE_FORMAT_ELEMENT_IDS)
|
|
_STORE_FORMAT_PREFIX = "cultcache.store."
|
|
_PERSISTED_RECORD_SLOTS = 4
|
|
_LOCK_POLL_SECONDS = 0.01
|
|
LOCK_TIMEOUT_SECONDS = 30.0
|
|
_Contents = TypeVar("_Contents")
|
|
|
|
|
|
class StoreUnreadableError(ValueError):
|
|
"""A store file this runtime cannot read: not exactly one complete store (truncated, bytes after it, a missing or extra
|
|
slot), a header or record it does not know, or a body it cannot decode. Open, push, delete and push_all refuse a file
|
|
with this error, and a refused file is left as it was. The cause is `__cause__`."""
|
|
|
|
def __init__(self, path: Path, cause: BaseException) -> None:
|
|
super().__init__(f"CultCache store {path} is not readable: {cause}")
|
|
self.path = path
|
|
|
|
|
|
class SchemaConflictError(ValueError):
|
|
"""A write the catalog cannot describe: records of different types under one schema id, two arrived schemas that share an id
|
|
and disagree on the schema name, or a record no chosen entry publishes. Nothing is written. It names the id, the names
|
|
involved (`schema_names`) and a record key (`record_key`). A cache raises it too, with an empty record key, when a
|
|
document registers a schema id another registered document already carries. `unread_slots` is None except for the refusal to
|
|
write or remove a record held read-only because its payload carries slots past the declared ones; its message names the type
|
|
and both slot counts and never holds a slot value."""
|
|
|
|
def __init__(
|
|
self,
|
|
schema_id: str,
|
|
schema_names: list[str],
|
|
record_key: str,
|
|
*,
|
|
message: str | None = None,
|
|
unread_slots: UnreadSlots | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
message
|
|
or f"Schema id {schema_id!r} cannot describe record {record_key!r}: {', '.join(repr(name) for name in schema_names)}"
|
|
)
|
|
self.schema_id = schema_id
|
|
self.schema_names = schema_names
|
|
self.record_key = record_key
|
|
self.unread_slots = unread_slots
|
|
|
|
|
|
class JsonLinesBackingStore:
|
|
def __init__(self, path: str | os.PathLike[str]) -> None:
|
|
self.path = Path(path)
|
|
self._lock = threading.RLock()
|
|
|
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
|
with self._lock:
|
|
text = _read_store(self.path, lambda path: path.read_text(encoding="utf-8"))
|
|
if text is None:
|
|
return []
|
|
envelopes: list[CultCacheEnvelope] = []
|
|
for line_number, line in enumerate(text.splitlines(), start=1):
|
|
if not line.strip():
|
|
continue
|
|
raw = json.loads(line)
|
|
try:
|
|
payload = base64.b64decode(raw["payload"])
|
|
envelopes.append(
|
|
CultCacheEnvelope(
|
|
key=raw["key"],
|
|
type=raw["type"],
|
|
payload=payload,
|
|
stored_at=raw.get("stored_at", raw.get("storedAt")),
|
|
schema_id=raw.get("schema_id", raw.get("schemaId")),
|
|
)
|
|
)
|
|
except KeyError as exc:
|
|
raise ValueError(f"Malformed CultCache JSONL envelope at {self.path}:{line_number}: missing {exc}") from exc
|
|
return envelopes
|
|
|
|
def push(self, envelope: CultCacheEnvelope) -> None:
|
|
with self._lock:
|
|
existing = {(item.type, item.key): item for item in self.pull_all()}
|
|
existing[(envelope.type, envelope.key)] = envelope
|
|
self._replace_all(list(existing.values()))
|
|
|
|
def push_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
|
with self._lock:
|
|
existing = {(item.type, item.key): item for item in self.pull_all()}
|
|
for envelope in envelopes:
|
|
existing[(envelope.type, envelope.key)] = envelope
|
|
self._replace_all(list(existing.values()))
|
|
|
|
def delete(self, type: str, key: str) -> None:
|
|
with self._lock:
|
|
existing = [item for item in self.pull_all() if not (item.type == type and item.key == key)]
|
|
self._replace_all(existing)
|
|
|
|
def _replace_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
temp = self.path.with_suffix(self.path.suffix + ".tmp")
|
|
lines = []
|
|
for envelope in sorted(envelopes, key=lambda item: (item.type, item.key)):
|
|
lines.append(json.dumps({
|
|
"key": envelope.key,
|
|
"type": envelope.type,
|
|
"payload": base64.b64encode(envelope.payload).decode("ascii"),
|
|
"stored_at": envelope.stored_at,
|
|
}, ensure_ascii=False, sort_keys=True))
|
|
temp.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
|
|
_replace_store(temp, self.path)
|
|
|
|
|
|
class SingleFileMessagePackBackingStore:
|
|
def __init__(self, path: str | os.PathLike[str], lock_timeout: float = LOCK_TIMEOUT_SECONDS) -> None:
|
|
self.path = Path(path)
|
|
self.lock_timeout = lock_timeout
|
|
self._lock = threading.RLock()
|
|
# The header this store read; a rewrite keeps it, so a store that holds element ids never sheds its marker.
|
|
self._format = STORE_FORMAT_VERSION
|
|
|
|
def pull_all(self) -> list[CultCacheEnvelope]:
|
|
with self._lock:
|
|
# An empty file is not a store (no writer leaves one), so it is decoded and refused.
|
|
# The disk decides the header: a file that is gone is not marked.
|
|
self._format = STORE_FORMAT_VERSION
|
|
data = _read_store(self.path, Path.read_bytes)
|
|
if data is None:
|
|
return []
|
|
try:
|
|
decoded = msgpack.unpackb(data, raw=False)
|
|
snapshot = _decode_snapshot(decoded)
|
|
if snapshot is not None:
|
|
self._format, envelopes = snapshot
|
|
return envelopes
|
|
return _decode_legacy_envelopes(decoded)
|
|
except Exception as exc:
|
|
raise StoreUnreadableError(self.path, exc) from exc
|
|
|
|
def push(self, envelope: CultCacheEnvelope) -> None:
|
|
with self._write_transaction():
|
|
self._replace_all([envelope], self.pull_all())
|
|
|
|
def push_all(self, envelopes: list[CultCacheEnvelope]) -> None:
|
|
with self._write_transaction():
|
|
self._replace_all(list(envelopes), self.pull_all())
|
|
|
|
def delete(self, type: str, key: str) -> None:
|
|
with self._write_transaction():
|
|
self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)])
|
|
|
|
@contextmanager
|
|
def _write_transaction(self) -> Iterator[None]:
|
|
"""The thread lock, then the cross-process lock on `<path>.lock`, held across the read and the replace so two
|
|
processes never rewrite from the same stale read."""
|
|
with self._lock:
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
with store_write_lock(self.path, self.lock_timeout):
|
|
yield
|
|
|
|
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None:
|
|
commit_store_file(self.path, msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True))
|
|
|
|
|
|
|
|
def canonical(candidates: Any) -> list[CultCacheSchemaCatalogEntry]:
|
|
"""Entries that tie, in one fixed order, so the catalog does not depend on the order they arrive in."""
|
|
return sorted(candidates, key=lambda entry: (entry.schema_name, entry.content_hash, ",".join(entry.compatible_schema_ids)))
|
|
|
|
|
|
def _derive_catalog(records: list[tuple[CultCacheEnvelope, bool]]) -> list[CultCacheSchemaCatalogEntry]:
|
|
"""The catalog a write leaves, derived from the records being written and nothing else. For each schema id a record carries,
|
|
one entry that publishes it: the entry that owns the id (a registered one, else one that arrived with the records), else one
|
|
that lists it as compatible (registered first). Entries are written as chosen, never merged; a registered entry wins over an
|
|
arrived one with the same own id. Records of different types under one id, two arrived entries with the same own id and
|
|
different schema names, or a record no chosen entry publishes, refuse the write. `records` are (envelope, registered): an
|
|
envelope the caller supplied is registered, one read back from the file is arrived."""
|
|
entries: list[tuple[CultCacheSchemaCatalogEntry, bool]] = []
|
|
by_id: dict[str, list[CultCacheEnvelope]] = {}
|
|
tiers: dict[str, list[tuple[CultCacheEnvelope, bool, CultCacheSchemaCatalogEntry]]] = {}
|
|
for envelope, registered in records:
|
|
schema_id = _schema_id_for(envelope)
|
|
supplied = envelope.catalog_entry
|
|
publishes = supplied is not None and (supplied.schema_id == schema_id or schema_id in supplied.compatible_schema_ids)
|
|
entry = supplied if publishes else _default_catalog_entry(envelope)
|
|
entries.append((entry, registered))
|
|
by_id.setdefault(schema_id, []).append(envelope)
|
|
tiers.setdefault(schema_id, []).append((envelope, registered, entry))
|
|
|
|
chosen: dict[str, tuple[CultCacheSchemaCatalogEntry, bool]] = {}
|
|
for schema_id in sorted(by_id):
|
|
carrying = by_id[schema_id]
|
|
record_key = min(envelope.key for envelope in carrying)
|
|
# Records of one tier under one id are one type. Across tiers the id keeps the type its records resolve to, so a write may
|
|
# not retype records that are already there: unless the entries of both tiers own the id, which is a rename of the schema
|
|
# under a stable id, and the registered descriptor wins. A read-back record's type is its schema's name, so tiers compare
|
|
# the names of the entries that publish the id.
|
|
for tier in (True, False):
|
|
types = sorted({envelope.type for envelope, registered, _ in tiers[schema_id] if registered == tier})
|
|
if len(types) > 1:
|
|
raise SchemaConflictError(schema_id, types, record_key)
|
|
registered_names = sorted({entry.schema_name for _, registered, entry in tiers[schema_id] if registered})
|
|
arrived_names = sorted({entry.schema_name for _, registered, entry in tiers[schema_id] if not registered})
|
|
owned_by = lambda tier: any(entry.schema_id == schema_id for _, registered, entry in tiers[schema_id] if registered == tier)
|
|
if registered_names and arrived_names and registered_names[0] != arrived_names[0] and not (owned_by(True) and owned_by(False)):
|
|
raise SchemaConflictError(schema_id, sorted([registered_names[0], arrived_names[0]]), record_key)
|
|
own_registered = next(iter(canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)), None)
|
|
own_arrived = canonical(entry for entry, registered in entries if not registered and entry.schema_id == schema_id)
|
|
own_registered_all = canonical(entry for entry, registered in entries if registered and entry.schema_id == schema_id)
|
|
# Entries of one tier that share an own id must be one schema; a registered entry settles a disagreement among arrived ones.
|
|
names = sorted({entry.schema_name for entry in (own_registered_all or own_arrived)})
|
|
if len(names) > 1:
|
|
raise SchemaConflictError(schema_id, names, record_key)
|
|
if own_registered is not None:
|
|
pick, registered_pick = own_registered, True
|
|
elif own_arrived:
|
|
pick, registered_pick = own_arrived[0], False
|
|
else:
|
|
listed_registered = next(iter(canonical(entry for entry, registered in entries if registered and schema_id in entry.compatible_schema_ids)), None)
|
|
listed_arrived = next(iter(canonical(entry for entry, registered in entries if not registered and schema_id in entry.compatible_schema_ids)), None)
|
|
pick = listed_registered if listed_registered is not None else listed_arrived
|
|
registered_pick = listed_registered is not None
|
|
if pick is None:
|
|
continue
|
|
prior = chosen.get(pick.schema_id)
|
|
if prior is not None and prior[1] and not registered_pick:
|
|
continue
|
|
chosen[pick.schema_id] = (pick, registered_pick)
|
|
|
|
for schema_id in sorted(by_id):
|
|
if not any(entry.schema_id == schema_id or schema_id in entry.compatible_schema_ids for entry, _ in chosen.values()):
|
|
raise SchemaConflictError(
|
|
schema_id, sorted(entry.schema_name for entry, _ in chosen.values()), min(envelope.key for envelope in by_id[schema_id])
|
|
)
|
|
return [entry for entry, _ in chosen.values()]
|
|
|
|
|
|
def retyped(held: CultCacheEnvelope, written: CultCacheEnvelope) -> SchemaConflictError:
|
|
"""The refusal of a write that would put a second type under a key the store already holds (C# CacheBackingStore.Retyped)."""
|
|
return SchemaConflictError(
|
|
_schema_id_for(held),
|
|
[held.type],
|
|
held.key,
|
|
message=(
|
|
f"Record '{held.key}' is a {held.type} (schema id '{_schema_id_for(held)}'); writing a {written.type} there would "
|
|
"retype the key. Remove the record first, in the same batch or an earlier commit, then add the other type's."
|
|
),
|
|
)
|
|
|
|
|
|
def written_owns(held: CultCacheEnvelope, written: CultCacheEnvelope) -> bool:
|
|
"""Whether a write of `written` may replace `held` at its key (C# WrittenOwns): the written type's entry declares the held
|
|
record's schema id (its own id or one it lists compatible), or it is the type of the same schema name. The same-name half
|
|
stands in for C#'s "reads the published entry whole": this runtime does not compare shapes, so it never refuses a
|
|
same-name type for dropping slots."""
|
|
stored_id = _schema_id_for(held)
|
|
entry = written.catalog_entry if written.catalog_entry is not None else _default_catalog_entry(written)
|
|
return stored_id == entry.schema_id or stored_id in entry.compatible_schema_ids or held.type == written.type
|
|
|
|
|
|
def _encode_snapshot(
|
|
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
|
|
) -> list[Any]:
|
|
# A key names one record whatever its type, so a supplied record replaces the arrived one by key alone, and only of the
|
|
# type that owns it: retyping a key is a removal and an add, so a record another type owns under the key refuses the write.
|
|
held_by_key = {envelope.key: envelope for envelope in arrived}
|
|
landing: dict[str, CultCacheEnvelope] = {}
|
|
for envelope in supplied:
|
|
for held in (landing.get(envelope.key), held_by_key.get(envelope.key)):
|
|
if held is not None and not written_owns(held, envelope):
|
|
raise retyped(held, envelope)
|
|
landing[envelope.key] = envelope
|
|
supplied = list(landing.values())
|
|
replaced = set(landing)
|
|
kept = [envelope for envelope in arrived if envelope.key not in replaced]
|
|
envelopes = [*kept, *supplied]
|
|
catalog = [
|
|
_encode_catalog_entry(entry)
|
|
for entry in sorted(
|
|
_derive_catalog([*((envelope, False) for envelope in kept), *((envelope, True) for envelope in supplied)]),
|
|
key=lambda item: (item.schema_name, item.schema_id),
|
|
)
|
|
]
|
|
records = [
|
|
[envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
|
|
for envelope in sorted(envelopes, key=lambda item: item.key)
|
|
]
|
|
return [format_version, catalog, records]
|
|
|
|
def commit_store_file(path: Path, data: bytes) -> None:
|
|
"""Writes `data` to a unique staged file beside the store, fsyncs it, replaces the store with it and, off Windows,
|
|
fsyncs the directory so the replace is durable. A failure before the replace removes the staged file and leaves the
|
|
store untouched. The caller holds the store lock."""
|
|
staged = path.with_name(f"{path.name}.tmp-{os.getpid()}-{uuid.uuid4().hex}")
|
|
try:
|
|
fd = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0), 0o666)
|
|
try:
|
|
view = memoryview(data)
|
|
while view:
|
|
view = view[os.write(fd, view):]
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
_replace_store(staged, path)
|
|
except BaseException:
|
|
try:
|
|
os.unlink(staged)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
if sys.platform != "win32":
|
|
directory = os.open(path.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory)
|
|
finally:
|
|
os.close(directory)
|
|
|
|
|
|
REPLACE_ATTEMPTS = 25
|
|
REPLACE_RETRY_SECONDS = 0.02
|
|
_ERROR_ACCESS_DENIED = 5
|
|
_ERROR_SHARING_VIOLATION = 32
|
|
_ERROR_LOCK_VIOLATION = 33
|
|
_POSIX_UNSUPPORTED = (1, 50, 87) # incorrect function, not supported, invalid parameter: FAT, exFAT, before Windows 10 1607
|
|
MOVEFILE_REPLACE_WRITE_THROUGH = 0x9 # MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH
|
|
|
|
|
|
class ReplaceKernel:
|
|
"""The kernel32 calls a Windows store replace needs, each answering 0 or a Win32 error code."""
|
|
|
|
def rename_posix(self, staged: str, destination: str) -> int:
|
|
"""CreateFileW(staged, DELETE|SYNCHRONIZE), then FileRenameInfoEx with REPLACE_IF_EXISTS|POSIX_SEMANTICS, then CloseHandle."""
|
|
raise NotImplementedError
|
|
|
|
def move_file_ex(self, staged: str, destination: str, flags: int) -> int:
|
|
raise NotImplementedError
|
|
|
|
def sleep(self, seconds: float) -> None:
|
|
time.sleep(seconds)
|
|
|
|
|
|
def _replace_store(staged: str | os.PathLike[str], destination: str | os.PathLike[str]) -> None:
|
|
"""Replaces `destination` with `staged`: POSIX-semantics rename first on Windows (so a reader holding the store open
|
|
never sees it absent), os.replace elsewhere."""
|
|
if sys.platform == "win32":
|
|
replace_store_windows(os.fspath(staged), os.fspath(destination), WindowsReplaceKernel())
|
|
else:
|
|
os.replace(staged, destination)
|
|
|
|
|
|
def replace_store_windows(staged: str, destination: str, kernel: ReplaceKernel) -> None:
|
|
"""POSIX-semantics rename, falling back for the rest of the call to write-through MoveFileExW where POSIX semantics are
|
|
unsupported; errors 5, 32 and 33 retry up to 25 times 20 ms apart, any other code raises at once. The TypeScript and C#
|
|
runtimes take the same decision table."""
|
|
posix = True
|
|
last = 0
|
|
for _ in range(REPLACE_ATTEMPTS):
|
|
code = kernel.rename_posix(staged, destination) if posix else kernel.move_file_ex(staged, destination, MOVEFILE_REPLACE_WRITE_THROUGH)
|
|
if posix and code in _POSIX_UNSUPPORTED:
|
|
posix = False
|
|
code = kernel.move_file_ex(staged, destination, MOVEFILE_REPLACE_WRITE_THROUGH)
|
|
if code == 0:
|
|
return
|
|
if code not in (_ERROR_ACCESS_DENIED, _ERROR_SHARING_VIOLATION, _ERROR_LOCK_VIOLATION):
|
|
raise _replace_error(f"CultCache could not replace the store: Win32 error {code}.", code)
|
|
last = code
|
|
kernel.sleep(REPLACE_RETRY_SECONDS)
|
|
raise _replace_error(f"CultCache could not replace the store after {REPLACE_ATTEMPTS} attempts: last Win32 error {last}.", last)
|
|
|
|
|
|
def _replace_error(message: str, code: int) -> OSError:
|
|
error = OSError(message)
|
|
error.winerror = code # type: ignore[attr-defined]
|
|
return error
|
|
|
|
|
|
class WindowsReplaceKernel(ReplaceKernel):
|
|
"""kernel32 bound by ctypes, as _try_lock_windows binds it. Windows only."""
|
|
|
|
def __init__(self) -> None:
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
|
|
self._ctypes = ctypes
|
|
k = self._k = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
k.CreateFileW.restype = wintypes.HANDLE
|
|
k.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p, wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE]
|
|
k.SetFileInformationByHandle.restype = wintypes.BOOL
|
|
k.SetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
|
|
k.CloseHandle.restype = wintypes.BOOL
|
|
k.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
k.MoveFileExW.restype = wintypes.BOOL
|
|
k.MoveFileExW.argtypes = [wintypes.LPCWSTR, wintypes.LPCWSTR, wintypes.DWORD]
|
|
|
|
def rename_posix(self, staged: str, destination: str) -> int:
|
|
ctypes, k = self._ctypes, self._k
|
|
delete_synchronize, share_all, open_existing, normal = 0x00110000, 0x7, 3, 0x80
|
|
handle = k.CreateFileW(staged, delete_synchronize, share_all, None, open_existing, normal, None)
|
|
if handle is None or ctypes.c_void_p(handle).value in (None, ctypes.c_void_p(-1).value):
|
|
return ctypes.get_last_error()
|
|
try:
|
|
# FILE_RENAME_INFO: Flags (DWORD), RootDirectory (pointer-aligned HANDLE), FileNameLength (DWORD), FileName (WCHAR[]).
|
|
name = os.path.abspath(destination).encode("utf-16-le")
|
|
pointer = ctypes.sizeof(ctypes.c_void_p)
|
|
length_at = pointer + pointer
|
|
info = ctypes.create_string_buffer(length_at + 4 + len(name) + 2)
|
|
info[0:4] = (0x3).to_bytes(4, "little") # FILE_RENAME_FLAG_REPLACE_IF_EXISTS | FILE_RENAME_FLAG_POSIX_SEMANTICS
|
|
info[length_at:length_at + 4] = len(name).to_bytes(4, "little")
|
|
info[length_at + 4:length_at + 4 + len(name)] = name
|
|
file_rename_info_ex = 22
|
|
if k.SetFileInformationByHandle(handle, file_rename_info_ex, info, len(info)):
|
|
return 0
|
|
return ctypes.get_last_error()
|
|
finally:
|
|
k.CloseHandle(handle)
|
|
|
|
def move_file_ex(self, staged: str, destination: str, flags: int) -> int:
|
|
return 0 if self._k.MoveFileExW(staged, destination, flags) else self._ctypes.get_last_error()
|
|
|
|
|
|
@contextmanager
|
|
def store_write_lock(path: Path, timeout: float = LOCK_TIMEOUT_SECONDS) -> Iterator[None]:
|
|
"""Holds the exclusive store lock on `<path>.lock`, the lock the C#, Rust and TypeScript runtimes take: flock on Unix,
|
|
LockFileEx over the whole file on Windows, polled every 10 ms with no handle held between polls. The lock file is
|
|
created if absent and never written or deleted."""
|
|
lock_path = path.with_name(path.name + ".lock")
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
release = _try_lock(lock_path)
|
|
if release is not None:
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError(f"CultCache timed out after {timeout} s waiting for the store lock {lock_path}.")
|
|
time.sleep(_LOCK_POLL_SECONDS)
|
|
try:
|
|
yield
|
|
finally:
|
|
release()
|
|
|
|
|
|
def _try_lock(lock_path: Path) -> Callable[[], None] | None:
|
|
if sys.platform == "win32":
|
|
return _try_lock_windows(lock_path)
|
|
import fcntl
|
|
|
|
fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o666)
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
os.close(fd)
|
|
return None
|
|
return lambda: os.close(fd) # closing drops the flock
|
|
|
|
|
|
def _try_lock_windows(lock_path: Path) -> Callable[[], None] | None:
|
|
import ctypes
|
|
import msvcrt
|
|
from ctypes import wintypes
|
|
|
|
kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
kernel.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, ctypes.c_void_p]
|
|
fd = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_BINARY, 0o666)
|
|
handle = msvcrt.get_osfhandle(fd)
|
|
overlapped = ctypes.create_string_buffer(32)
|
|
# LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY over the whole range, as the TypeScript runtime takes it.
|
|
if not kernel.LockFileEx(handle, 0x3, 0, 0xFFFFFFFF, 0xFFFFFFFF, overlapped):
|
|
os.close(fd)
|
|
return None
|
|
return lambda: os.close(fd) # closing drops the lock
|
|
|
|
|
|
def _read_store(path: Path, read: Callable[[Path], _Contents]) -> _Contents | None:
|
|
"""Reads a store file, or returns None when nothing is at its path.
|
|
|
|
Only nothing is absent. A dangling symbolic link is something: reading it as
|
|
empty would let the writer replace the link and move the store off its volume.
|
|
Any other failure to reach the file stays an OSError.
|
|
"""
|
|
try:
|
|
return read(path)
|
|
except FileNotFoundError:
|
|
if path.is_symlink():
|
|
raise
|
|
return None
|
|
|
|
|
|
def _describe_identity(value: Any) -> str:
|
|
"""A record key or schema id as a refusal may show it: a string is an identity and is
|
|
named; anything else is described by its type, since the value is the store's."""
|
|
if isinstance(value, str):
|
|
return repr(value)
|
|
return f"<{type(value).__name__}>"
|
|
|
|
|
|
def _describe_header(header: str) -> str:
|
|
"""Echoes a header only in the shape cultcache.store.v<digits>; the bytes are the store's."""
|
|
version = header[len(_STORE_FORMAT_PREFIX) + 1:] if header.startswith(_STORE_FORMAT_PREFIX + "v") else ""
|
|
if version.isascii() and version.isdigit():
|
|
return repr(header)
|
|
return f"an unrecognised {_STORE_FORMAT_PREFIX}* header of {len(header.encode('utf-8'))} bytes"
|
|
|
|
|
|
def _decode_snapshot(decoded: Any) -> tuple[str, list[CultCacheEnvelope]] | None:
|
|
if not isinstance(decoded, list) or not decoded:
|
|
return None
|
|
if isinstance(decoded[0], str) and decoded[0].startswith(_STORE_FORMAT_PREFIX) and decoded[0] not in _READABLE_STORE_FORMATS:
|
|
raise ValueError(
|
|
f"CultCache store format {_describe_header(decoded[0])} is not one this runtime reads; "
|
|
f"it reads {STORE_FORMAT_VERSION!r} and {STORE_FORMAT_ELEMENT_IDS!r} only"
|
|
)
|
|
if decoded[0] not in _READABLE_STORE_FORMATS:
|
|
return None
|
|
if len(decoded) != 3:
|
|
raise ValueError(f"CultCache store has {len(decoded)} top-level slots; this runtime reads 3 (header, schema catalog, records)")
|
|
if not isinstance(decoded[1], list) or not isinstance(decoded[2], list):
|
|
raise ValueError("CultCache v1 snapshot must contain a schema catalog and record array")
|
|
|
|
# An id names the entry that owns it; only an id no entry owns names the entry that lists it as compatible.
|
|
entries = [_decode_catalog_entry(raw_entry) for raw_entry in decoded[1]]
|
|
catalog_by_schema_id: dict[str, CultCacheSchemaCatalogEntry] = {}
|
|
for entry in entries:
|
|
catalog_by_schema_id.setdefault(entry.schema_id, entry)
|
|
for entry in entries:
|
|
for compatible_schema_id in entry.compatible_schema_ids:
|
|
catalog_by_schema_id.setdefault(compatible_schema_id, entry)
|
|
|
|
envelopes: list[CultCacheEnvelope] = []
|
|
seen: set[str] = set()
|
|
for raw_record in decoded[2]:
|
|
if not isinstance(raw_record, list) or len(raw_record) < 4:
|
|
raise ValueError("CultCache persisted records must be MessagePack arrays")
|
|
if len(raw_record) > _PERSISTED_RECORD_SLOTS:
|
|
raise ValueError(
|
|
f"CultCache record {_describe_identity(raw_record[0])} (schema {_describe_identity(raw_record[1])}) "
|
|
f"has {len(raw_record)} slots, more than "
|
|
f"the {_PERSISTED_RECORD_SLOTS} of a {STORE_FORMAT_VERSION} record, so this is not a valid store"
|
|
)
|
|
key, schema_id, stored_at, payload = raw_record
|
|
if not isinstance(key, str) or not key:
|
|
raise ValueError("CultCache persisted records must declare a key")
|
|
if key in seen:
|
|
raise ValueError(f"CultCache record {key!r} appears more than once")
|
|
seen.add(key)
|
|
if not isinstance(schema_id, str) or not schema_id:
|
|
raise ValueError("CultCache persisted records must declare a schema id")
|
|
if not isinstance(stored_at, str) or not stored_at:
|
|
raise ValueError("CultCache persisted records must declare storedAt")
|
|
catalog_entry = catalog_by_schema_id.get(schema_id)
|
|
if catalog_entry is None:
|
|
raise ValueError(f"CultCache persisted record '{key}' references schema id '{schema_id}', which the store's catalog does not publish")
|
|
normalized_payload = _normalize_payload(payload)
|
|
envelopes.append(
|
|
CultCacheEnvelope(
|
|
key=key,
|
|
type=catalog_entry.schema_name,
|
|
payload=normalized_payload,
|
|
stored_at=stored_at,
|
|
schema_id=schema_id,
|
|
catalog_entry=catalog_entry,
|
|
)
|
|
)
|
|
return decoded[0], envelopes
|
|
|
|
|
|
def _decode_legacy_envelopes(decoded: Any) -> list[CultCacheEnvelope]:
|
|
if not isinstance(decoded, list):
|
|
raise ValueError("CultCache MessagePack store is not a recognized snapshot")
|
|
envelopes: list[CultCacheEnvelope] = []
|
|
seen: set[Any] = set()
|
|
for item in decoded:
|
|
if not isinstance(item, dict):
|
|
raise ValueError("CultCache legacy envelopes must be MessagePack maps")
|
|
if item.get("key") in seen:
|
|
raise ValueError(f"CultCache record {item['key']!r} appears more than once")
|
|
seen.add(item.get("key"))
|
|
try:
|
|
payload = item["payload"]
|
|
envelopes.append(
|
|
CultCacheEnvelope(
|
|
key=item["key"],
|
|
type=item["type"],
|
|
payload=payload if isinstance(payload, bytes) else msgpack.packb(payload, use_bin_type=True),
|
|
stored_at=item.get("storedAt", item.get("stored_at")),
|
|
schema_id=item.get("schemaId", item.get("schema_id")),
|
|
)
|
|
)
|
|
except KeyError as exc:
|
|
raise ValueError(f"Malformed CultCache legacy envelope: missing {exc}") from exc
|
|
return envelopes
|
|
|
|
|
|
def _encode_catalog_entry(entry: CultCacheSchemaCatalogEntry) -> list[Any]:
|
|
return [
|
|
entry.schema_id,
|
|
entry.schema_name,
|
|
entry.schema_version,
|
|
entry.content_hash,
|
|
entry.canonical_schema_json,
|
|
list(entry.compatible_schema_ids or (entry.schema_id,)),
|
|
[_encode_catalog_member(member) for member in entry.members],
|
|
]
|
|
|
|
|
|
def _decode_catalog_entry(value: Any) -> CultCacheSchemaCatalogEntry:
|
|
if not isinstance(value, list):
|
|
raise ValueError("CultCache schema catalog entries must be MessagePack arrays")
|
|
schema_id = value[0] if len(value) > 0 else ""
|
|
schema_name = value[1] if len(value) > 1 else ""
|
|
schema_version = value[2] if len(value) > 2 else ""
|
|
content_hash = value[3] if len(value) > 3 else ""
|
|
canonical_schema_json = value[4] if len(value) > 4 else ""
|
|
compatible_schema_ids = value[5] if len(value) > 5 else []
|
|
members = value[6] if len(value) > 6 else []
|
|
if not isinstance(schema_id, str) or not schema_id or not isinstance(schema_name, str) or not schema_name:
|
|
raise ValueError("CultCache schema catalog entries must declare schemaId and schemaName")
|
|
if not isinstance(compatible_schema_ids, list) or not all(isinstance(item, str) and item for item in compatible_schema_ids):
|
|
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid compatible schema ids')
|
|
if not isinstance(members, list):
|
|
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has invalid members')
|
|
if not isinstance(canonical_schema_json, str):
|
|
raise ValueError(f'CultCache schema catalog entry "{schema_id}" has an invalid canonical schema')
|
|
return CultCacheSchemaCatalogEntry(
|
|
schema_id=schema_id,
|
|
schema_name=schema_name,
|
|
schema_version=schema_version if isinstance(schema_version, str) and schema_version else f"{schema_name}.v1",
|
|
content_hash=content_hash if isinstance(content_hash, str) and content_hash else schema_id,
|
|
canonical_schema_json=canonical_schema_json,
|
|
compatible_schema_ids=tuple(compatible_schema_ids),
|
|
members=tuple(_decode_catalog_member(member) for member in members),
|
|
)
|
|
|
|
|
|
def _encode_catalog_member(member: CultCacheSchemaCatalogMember) -> list[Any]:
|
|
return [
|
|
member.slot,
|
|
member.member_name,
|
|
member.type_name,
|
|
member.is_reference,
|
|
member.is_many,
|
|
member.target_schema_name,
|
|
member.is_name,
|
|
member.index_alias,
|
|
]
|
|
|
|
|
|
def _decode_catalog_member(value: Any) -> CultCacheSchemaCatalogMember:
|
|
if not isinstance(value, list):
|
|
raise ValueError("CultCache schema catalog members must be MessagePack arrays")
|
|
slot = value[0] if len(value) > 0 else -1
|
|
member_name = value[1] if len(value) > 1 else ""
|
|
type_name = value[2] if len(value) > 2 else ""
|
|
if not isinstance(slot, int) or slot < 0 or not isinstance(member_name, str) or not member_name:
|
|
raise ValueError("CultCache schema catalog member has invalid slot or name")
|
|
if not isinstance(type_name, str) or not type_name:
|
|
raise ValueError("CultCache schema catalog member has invalid type")
|
|
return CultCacheSchemaCatalogMember(
|
|
slot=slot,
|
|
member_name=member_name,
|
|
type_name=type_name,
|
|
is_reference=(value[3] if len(value) > 3 else False) is True,
|
|
is_many=(value[4] if len(value) > 4 else False) is True,
|
|
target_schema_name=value[5] if len(value) > 5 and isinstance(value[5], str) else None,
|
|
is_name=(value[6] if len(value) > 6 else False) is True,
|
|
index_alias=value[7] if len(value) > 7 and isinstance(value[7], str) else None,
|
|
)
|
|
|
|
|
|
def _schema_id_for(envelope: CultCacheEnvelope) -> str:
|
|
return envelope.schema_id or envelope.type
|
|
|
|
|
|
def _default_catalog_entry(envelope: CultCacheEnvelope) -> CultCacheSchemaCatalogEntry:
|
|
schema_id = _schema_id_for(envelope)
|
|
return CultCacheSchemaCatalogEntry(
|
|
schema_id=schema_id,
|
|
schema_name=envelope.type,
|
|
schema_version=f"{envelope.type}.v1",
|
|
content_hash=schema_id,
|
|
canonical_schema_json=json.dumps(
|
|
{"schemaName": envelope.type, "schemaVersion": f"{envelope.type}.v1", "members": []},
|
|
separators=(",", ":"),
|
|
),
|
|
compatible_schema_ids=(schema_id,),
|
|
)
|
|
|
|
|
|
def _normalize_payload(payload: Any) -> bytes:
|
|
if isinstance(payload, bytes):
|
|
return payload
|
|
if isinstance(payload, bytearray):
|
|
return bytes(payload)
|
|
if isinstance(payload, list) and all(isinstance(item, int) and 0 <= item <= 255 for item in payload):
|
|
return bytes(payload)
|
|
raise ValueError("CultCache record payload must be binary MessagePack bytes")
|