cultcache-py: conditional update, content version, store-minted stored_at (update-with-retry-py)
Replaces the blind update and update_global with update(document, key, apply, max_attempts) over store read and commit_if; push and push_all restamp a replacing write later and report the stamp. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
afab2d5a12
commit
f954fdd595
7 changed files with 304 additions and 48 deletions
|
|
@ -1,7 +1,7 @@
|
|||
from .backing_store import BackingStore, CultCacheEnvelope
|
||||
from .backing_store import CultCacheSchemaCatalogEntry, CultCacheSchemaCatalogMember
|
||||
from . import leaves
|
||||
from .cache import CultCache, CultCacheBuilder, ForeignRecord
|
||||
from .cache import CultCache, CultCacheBuilder, ForeignRecord, UpdateOutcome, UpdateResult
|
||||
from .documents import (
|
||||
MAX_PAYLOAD_DEPTH,
|
||||
MAX_SLOT_KEY,
|
||||
|
|
@ -42,6 +42,8 @@ __all__ = [
|
|||
"StoreUnreadableError",
|
||||
"UnionArm",
|
||||
"UnreadSlots",
|
||||
"UpdateOutcome",
|
||||
"UpdateResult",
|
||||
"read_payload_tree",
|
||||
"leaves",
|
||||
"database_entry_field",
|
||||
|
|
|
|||
|
|
@ -58,10 +58,17 @@ class CultCacheEnvelope:
|
|||
|
||||
|
||||
class BackingStore(Protocol):
|
||||
"""The members every store has. A store may also offer `read(key) -> tuple[CultCacheEnvelope, str] | None` (the record at the
|
||||
key, of any type, and its content version, from one read) and `commit_if(envelope, expected: str | None) -> str | None` (write
|
||||
only if the record at the key is still the version `expected`, None meaning no record; returns the stored_at written, or None
|
||||
with nothing written). They are looked up with getattr, not Protocol members, and `CultCache.update` needs both. `push` and
|
||||
`push_all` may report the stored_at they wrote (`str` and `dict[str, str]` by key); None keeps the cache's own stamp. A store
|
||||
stamps a record that replaces another strictly later than the one it replaces."""
|
||||
|
||||
def pull_all(self) -> list[CultCacheEnvelope]:
|
||||
...
|
||||
|
||||
def push(self, envelope: CultCacheEnvelope) -> None:
|
||||
def push(self, envelope: CultCacheEnvelope) -> str | None:
|
||||
...
|
||||
|
||||
def delete(self, type: str, key: str) -> None:
|
||||
|
|
@ -74,7 +81,7 @@ class BackingStore(Protocol):
|
|||
that can tell nothing-at-the-path overrides this, and the default push_all then writes where nothing is on a soft push."""
|
||||
return self.pull_all()
|
||||
|
||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None:
|
||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> dict[str, str] | None:
|
||||
# Soft: write only where pull_or_absent says nothing is; skip over a store it read, empty included; its raise refuses.
|
||||
pulled = self.pull_or_absent()
|
||||
if soft and pulled is not None:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import functools
|
||||
import math
|
||||
import threading
|
||||
|
|
@ -9,13 +10,39 @@ from typing import Any, Callable, Generic, Iterable, TypeVar
|
|||
|
||||
from .backing_store import BackingStore, CultCacheEnvelope
|
||||
from .documents import DocumentDefinition, UnreadSlots, extract_value
|
||||
from .stores import SchemaConflictError, retyped, written_owns
|
||||
from .stores import SchemaConflictError, _parse_stored_at, retyped, stored_at_over, written_owns
|
||||
|
||||
T = TypeVar("T")
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
GLOBAL_KEY = "__global__"
|
||||
|
||||
|
||||
class UpdateOutcome(enum.Enum):
|
||||
UPDATED = "updated"
|
||||
DECLINED = "declined"
|
||||
LOST_RACE = "lost_race"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UpdateResult(Generic[T]):
|
||||
"""What `CultCache.update` did and the cache's record at the key after it (None when it holds nothing). The document decides
|
||||
nothing: it is a read after the outcome."""
|
||||
|
||||
outcome: UpdateOutcome
|
||||
document: T | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Expected:
|
||||
"""What an update's commit requires to still hold. A cache with no store requires the held envelope object at the key (None:
|
||||
nothing held); a store-backed one requires the type's home to be `store` and the record there to have content version
|
||||
`version` (None: no record)."""
|
||||
|
||||
held: CultCacheEnvelope | None = None
|
||||
store: BackingStore | None = None
|
||||
version: str | None = None
|
||||
|
||||
|
||||
def _locked(method: F) -> F:
|
||||
"""Runs a read holding the cache's lock. Reads are reentrant and may run inside a mutation."""
|
||||
|
||||
|
|
@ -482,7 +509,7 @@ class CultCache:
|
|||
def put(self, document: DocumentDefinition[T], key: str, value: T) -> None:
|
||||
self._put(document, key, value)
|
||||
|
||||
def _put(self, document: DocumentDefinition[T], key: str, value: T) -> None:
|
||||
def _put(self, document: DocumentDefinition[T], key: str, value: T, expected: _Expected | None = None) -> str:
|
||||
self._assert_registered(document)
|
||||
if document.global_document and key != GLOBAL_KEY:
|
||||
raise CultCacheError(f"Global document {document.type} must use key {GLOBAL_KEY}")
|
||||
|
|
@ -494,7 +521,7 @@ class CultCache:
|
|||
schema_id=catalog_entry.schema_id,
|
||||
catalog_entry=catalog_entry,
|
||||
)
|
||||
self._write(document, [(envelope, value)], batch=False)
|
||||
return self._write(document, [(envelope, value)], batch=False, expected=expected)
|
||||
|
||||
@_mutating
|
||||
def put_envelope(self, document: DocumentDefinition[T], envelope: CultCacheEnvelope) -> T:
|
||||
|
|
@ -562,9 +589,12 @@ class CultCache:
|
|||
*,
|
||||
batch: bool,
|
||||
unread: dict[str, UnreadSlots | None] | None = None,
|
||||
) -> None:
|
||||
expected: _Expected | None = None,
|
||||
) -> str:
|
||||
"""Validates every record (home and name and index extractors included) before the store is
|
||||
touched; once the store accepts, applying to the cache has no fallible step."""
|
||||
touched; once the store accepts, applying to the cache has no fallible step. With `expected` (an update's commit) the write
|
||||
also requires the record to be unchanged since it was read and returns 'mismatch', having written and installed nothing,
|
||||
when it is not; otherwise 'committed'. A record is held under the stored_at its store reports writing."""
|
||||
type = document.type
|
||||
# A batch naming a key twice lands the last record for it, as C# CultCacheBatch stages one operation per key. The
|
||||
# judge, the store and the apply all read this one list.
|
||||
|
|
@ -584,13 +614,34 @@ class CultCache:
|
|||
)
|
||||
store = self._store_for_type(type)
|
||||
legacy_key = self._state.legacy_global_keys.get(type)
|
||||
if store is not None:
|
||||
if batch:
|
||||
store.push_all([envelope for envelope, _ in records])
|
||||
stamps: dict[str, Any] = {}
|
||||
if expected is not None:
|
||||
if legacy_key is not None:
|
||||
raise self._legacy_global_refusal(document, legacy_key)
|
||||
if expected.store is None:
|
||||
held = self._state.envelopes.get(type, {}).get(records[0][0].key)
|
||||
if store is not None or held is not expected.held:
|
||||
return "mismatch"
|
||||
elif store is not expected.store:
|
||||
return "mismatch"
|
||||
else:
|
||||
store.push(records[0][0])
|
||||
stamp = store.commit_if(records[0][0], expected.version) # type: ignore[attr-defined]
|
||||
if stamp is None:
|
||||
return "mismatch"
|
||||
stamps[records[0][0].key] = stamp
|
||||
elif store is not None:
|
||||
if batch:
|
||||
reported = store.push_all([envelope for envelope, _ in records])
|
||||
stamps = reported if isinstance(reported, dict) else {}
|
||||
else:
|
||||
stamps[records[0][0].key] = store.push(records[0][0])
|
||||
if legacy_key is not None:
|
||||
self._retire_legacy_global(store, type, legacy_key)
|
||||
else:
|
||||
for envelope, _ in records:
|
||||
held_at = next((held for held in (by_type.get(envelope.key) for by_type in self._state.envelopes.values()) if held is not None), None)
|
||||
if held_at is not None:
|
||||
stamps[envelope.key] = stored_at_over(envelope.stored_at, held_at.stored_at)
|
||||
self._state.legacy_global_keys.pop(type, None)
|
||||
values = self._state.values.setdefault(type, {})
|
||||
envelopes = self._state.envelopes.setdefault(type, {})
|
||||
|
|
@ -598,6 +649,9 @@ class CultCache:
|
|||
for other, key in displaced:
|
||||
self._drop(other, key)
|
||||
for envelope, value, keys in admitted:
|
||||
stamp = stamps.get(envelope.key)
|
||||
if isinstance(stamp, str) and _parse_stored_at(stamp) is not None:
|
||||
envelope = replace(envelope, stored_at=stamp)
|
||||
self._remove_lookups(type, envelope.key)
|
||||
values[envelope.key] = value
|
||||
envelopes[envelope.key] = envelope
|
||||
|
|
@ -663,18 +717,34 @@ class CultCache:
|
|||
if held is None:
|
||||
continue
|
||||
type, slots = held
|
||||
raise SchemaConflictError(
|
||||
self._state.envelopes[type][key].schema_id or type,
|
||||
[self._state.documents[type].catalog_entry().schema_name],
|
||||
key,
|
||||
message=(
|
||||
f"{self._state.documents[type].catalog_entry().schema_name} record {key} is read-only: {slots.type_name} was read with "
|
||||
f"{slots.slot_count} slots where this build declares {slots.declared_count} ({slots.occurrences} object(s) in the "
|
||||
"record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or "
|
||||
"write it from that build."
|
||||
),
|
||||
unread_slots=slots,
|
||||
)
|
||||
raise self._read_only_refusal(self._state.documents[type], key, self._state.envelopes[type][key].schema_id, slots)
|
||||
|
||||
@staticmethod
|
||||
def _read_only_refusal(
|
||||
document: DocumentDefinition[Any], key: str, schema_id: str | None, slots: UnreadSlots
|
||||
) -> SchemaConflictError:
|
||||
"""The one refusal of a write to a record that carries slots this build would drop: a record the cache holds read-only,
|
||||
or one an update reads from the store."""
|
||||
name = document.catalog_entry().schema_name
|
||||
return SchemaConflictError(
|
||||
schema_id or document.type,
|
||||
[name],
|
||||
key,
|
||||
message=(
|
||||
f"{name} record {key} is read-only: {slots.type_name} was read with "
|
||||
f"{slots.slot_count} slots where this build declares {slots.declared_count} ({slots.occurrences} object(s) in the "
|
||||
"record). Writing it would drop data a newer writer stored; read it with types that declare those slots, or "
|
||||
"write it from that build."
|
||||
),
|
||||
unread_slots=slots,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _legacy_global_refusal(document: DocumentDefinition[Any], legacy_key: str) -> CultCacheError:
|
||||
return CultCacheError(
|
||||
f"Global {document.type} is held under the legacy key {legacy_key!r}; update cannot copy it. "
|
||||
"put_global writes the global and retires the legacy key."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _foreign_refusal(record: ForeignRecord) -> SchemaConflictError:
|
||||
|
|
@ -707,20 +777,81 @@ class CultCache:
|
|||
self._assert_global(document)
|
||||
self._put(document, GLOBAL_KEY, value)
|
||||
|
||||
@_mutating
|
||||
def update(self, document: DocumentDefinition[T], key: str, updater: Any) -> T:
|
||||
return self._update(document, key, updater)
|
||||
def update(
|
||||
self, document: DocumentDefinition[T], key: str, apply: Callable[[T | None], T | None], *, max_attempts: int = 5
|
||||
) -> UpdateResult[T]:
|
||||
"""Reads the record at the key as its home store holds it, applies `apply` to a fresh copy of it, and commits the result
|
||||
only if the record is still the one that was read; when another writer got there first it reads again and runs `apply`
|
||||
again, up to `max_attempts` times.
|
||||
|
||||
def _update(self, document: DocumentDefinition[T], key: str, updater: Any) -> T:
|
||||
current = self.get_required(document, key)
|
||||
updated = updater(current)
|
||||
self._put(document, key, updated)
|
||||
return updated
|
||||
`apply` may run up to `max_attempts` times, on a fresh copy of the record as the store holds it now (None when the key
|
||||
holds nothing), which it may change and return. It must not write the same key through this cache or have side effects.
|
||||
Returning None writes nothing (DECLINED). A lost race that outlasts `max_attempts` returns LOST_RACE and does not raise;
|
||||
a refusal, or a raise from `apply`, propagates and is not retried. The write is durable when update returns. The
|
||||
result's document is the cache's record after the outcome. `put` is the blind write. A cache with no store copies its
|
||||
held instance; a store without `read` and `commit_if` (the JSON lines store among them) refuses before `apply` runs."""
|
||||
if not callable(apply):
|
||||
raise TypeError("CultCache update needs a function to apply.")
|
||||
if not isinstance(max_attempts, int) or isinstance(max_attempts, bool):
|
||||
raise TypeError(f"CultCache update max_attempts must be an int, not {type(max_attempts).__name__}.")
|
||||
if max_attempts < 1:
|
||||
raise ValueError(f"CultCache update max_attempts must be at least 1, not {max_attempts}.")
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
copy, expected = self._read_for_update(document, key)
|
||||
# Outside the cache's lock and the store's: apply may take its time and may write other keys through this cache.
|
||||
next_value = apply(copy)
|
||||
if next_value is None:
|
||||
return UpdateResult(UpdateOutcome.DECLINED, self.get(document, key))
|
||||
if self._commit_update(document, key, next_value, expected) == "committed":
|
||||
return UpdateResult(UpdateOutcome.UPDATED, self.get(document, key))
|
||||
# Display only: the cache shows what the winner left.
|
||||
if self._state.stores_by_type or self._state.generic_store is not None:
|
||||
self.pull_all_backing_stores()
|
||||
return UpdateResult(UpdateOutcome.LOST_RACE, self.get(document, key))
|
||||
|
||||
@_mutating
|
||||
def update_global(self, document: DocumentDefinition[T], updater: Any) -> T:
|
||||
def _read_for_update(self, document: DocumentDefinition[T], key: str) -> tuple[T | None, _Expected]:
|
||||
"""One attempt's read: the copy to hand `apply` and the condition its commit holds. Every refusal that does not depend on
|
||||
what `apply` returns is raised here, so `apply` never runs for a write that cannot land. Installs nothing."""
|
||||
self._assert_registered(document)
|
||||
self._refuse_at(document, key)
|
||||
legacy_key = self._state.legacy_global_keys.get(document.type)
|
||||
if legacy_key is not None:
|
||||
raise self._legacy_global_refusal(document, legacy_key)
|
||||
store = self._store_for_type(document.type)
|
||||
if store is None:
|
||||
held = self._state.envelopes.get(document.type, {}).get(key)
|
||||
return (None if held is None else document.decode_payload(held.payload)), _Expected(held=held)
|
||||
read, commit_if = getattr(store, "read", None), getattr(store, "commit_if", None)
|
||||
if read is None or commit_if is None:
|
||||
raise CultCacheError(
|
||||
f"Backing store {type(store).__name__} for {document.type} cannot write conditionally; "
|
||||
"update needs a store with read and commit_if."
|
||||
)
|
||||
if document.global_document:
|
||||
for envelope in store.pull_all():
|
||||
if envelope.key != GLOBAL_KEY and self._resolve_document_for_envelope(envelope) is document:
|
||||
raise self._legacy_global_refusal(document, envelope.key)
|
||||
found = read(key)
|
||||
if found is None:
|
||||
return None, _Expected(store=store)
|
||||
envelope, version = found
|
||||
if self._resolve_document_for_envelope(envelope) is not document:
|
||||
raise CultCacheError(f"Record {key} is not a {document.type}; update cannot copy it.")
|
||||
value, unread = self._decode_marked(document, envelope)
|
||||
if unread is not None:
|
||||
raise self._read_only_refusal(document, key, envelope.schema_id, unread)
|
||||
return value, _Expected(store=store, version=version)
|
||||
|
||||
@_mutating
|
||||
def _commit_update(self, document: DocumentDefinition[T], key: str, value: T, expected: _Expected) -> str:
|
||||
return self._put(document, key, value, expected)
|
||||
|
||||
def update_global(
|
||||
self, document: DocumentDefinition[T], apply: Callable[[T | None], T | None], *, max_attempts: int = 5
|
||||
) -> UpdateResult[T]:
|
||||
self._assert_global(document)
|
||||
return self._update(document, GLOBAL_KEY, updater)
|
||||
return self.update(document, GLOBAL_KEY, apply, max_attempts=max_attempts)
|
||||
|
||||
@_mutating
|
||||
def delete(self, document: DocumentDefinition[Any], key: str) -> None:
|
||||
|
|
|
|||
|
|
@ -1,13 +1,17 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterator, TypeVar
|
||||
|
||||
|
|
@ -107,14 +111,23 @@ class JsonLinesBackingStore:
|
|||
raise ValueError(f"Malformed CultCache JSONL envelope at {self.path}:{line_number}: missing {exc}") from exc
|
||||
return envelopes
|
||||
|
||||
def push(self, envelope: CultCacheEnvelope) -> None:
|
||||
self.push_all([envelope])
|
||||
def push(self, envelope: CultCacheEnvelope) -> str | None:
|
||||
return self.push_all([envelope])[envelope.key]
|
||||
|
||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> dict[str, str]:
|
||||
"""Merges the envelopes, each stored later than any record it replaces, and returns the stamps written by key
|
||||
({} when a soft push skipped)."""
|
||||
stamps: dict[str, str] = {}
|
||||
|
||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None:
|
||||
def merge(existing: dict[tuple[str, str], CultCacheEnvelope]) -> None:
|
||||
for envelope in envelopes:
|
||||
replaced = existing.get((envelope.type, envelope.key))
|
||||
if replaced is not None:
|
||||
envelope = replace(envelope, stored_at=stored_at_over(envelope.stored_at, replaced.stored_at))
|
||||
existing[(envelope.type, envelope.key)] = envelope
|
||||
stamps[envelope.key] = envelope.stored_at
|
||||
self._rewrite(merge, soft)
|
||||
return stamps
|
||||
|
||||
def delete(self, type: str, key: str) -> None:
|
||||
self._rewrite(lambda existing: existing.pop((type, key), None))
|
||||
|
|
@ -177,23 +190,40 @@ class SingleFileMessagePackBackingStore:
|
|||
except Exception as exc:
|
||||
raise StoreUnreadableError(self.path, exc) from exc
|
||||
|
||||
def push(self, envelope: CultCacheEnvelope) -> None:
|
||||
def push(self, envelope: CultCacheEnvelope) -> str | None:
|
||||
with self._write_transaction():
|
||||
self._replace_all([envelope], self.pull_all())
|
||||
return self._replace_all([envelope], self.pull_all())[envelope.key]
|
||||
|
||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> None:
|
||||
def push_all(self, envelopes: list[CultCacheEnvelope], soft: bool = False) -> dict[str, str]:
|
||||
"""A soft push writes only where nothing is at the path: a store the reader reads is left byte for byte, and a file it
|
||||
cannot read is refused by the same read, as in TS and Rust."""
|
||||
cannot read is refused by the same read, as in TS and Rust. Returns the stamps written by key ({} when skipped)."""
|
||||
with self._write_transaction():
|
||||
arrived = self._pull()
|
||||
if soft and arrived is not None:
|
||||
return
|
||||
self._replace_all(list(envelopes), arrived or [])
|
||||
return {}
|
||||
return self._replace_all(list(envelopes), arrived or [])
|
||||
|
||||
def delete(self, type: str, key: str) -> None:
|
||||
with self._write_transaction():
|
||||
self._replace_all([], [item for item in self.pull_all() if not (item.type == type and item.key == key)])
|
||||
|
||||
def read(self, key: str) -> tuple[CultCacheEnvelope, str] | None:
|
||||
"""The record at the key, of any type, and its content version, from one read; None when the key holds nothing."""
|
||||
for envelope in self._pull() or []:
|
||||
if envelope.key == key:
|
||||
return envelope, record_version(envelope)
|
||||
return None
|
||||
|
||||
def commit_if(self, envelope: CultCacheEnvelope, expected: str | None) -> str | None:
|
||||
"""Writes the envelope only if the record at its key (any type) is still the one whose version is `expected`
|
||||
(None: no record), judged under the write lock; returns the stored_at written, or None with nothing written."""
|
||||
with self._write_transaction():
|
||||
arrived = self._pull() or []
|
||||
current = next((item for item in arrived if item.key == envelope.key), None)
|
||||
if (None if current is None else record_version(current)) != expected:
|
||||
return None
|
||||
return self._replace_all([envelope], arrived)[envelope.key]
|
||||
|
||||
@contextmanager
|
||||
def _write_transaction(self) -> Iterator[None]:
|
||||
"""The thread lock, then the cross-process lock on `<path>.lock`, held across the read and the replace so two
|
||||
|
|
@ -203,8 +233,10 @@ class SingleFileMessagePackBackingStore:
|
|||
with store_write_lock(self.path, self.lock_timeout):
|
||||
yield
|
||||
|
||||
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> None:
|
||||
commit_store_file(self.path, msgpack.packb(_encode_snapshot(supplied, arrived, self._format), use_bin_type=True))
|
||||
def _replace_all(self, supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope]) -> dict[str, str]:
|
||||
snapshot, stamps = _encode_snapshot(supplied, arrived, self._format)
|
||||
commit_store_file(self.path, msgpack.packb(snapshot, use_bin_type=True))
|
||||
return stamps
|
||||
|
||||
|
||||
|
||||
|
|
@ -305,7 +337,7 @@ def written_owns(held: CultCacheEnvelope, written: CultCacheEnvelope) -> bool:
|
|||
|
||||
def _encode_snapshot(
|
||||
supplied: list[CultCacheEnvelope], arrived: list[CultCacheEnvelope], format_version: str
|
||||
) -> list[Any]:
|
||||
) -> tuple[list[Any], dict[str, str]]:
|
||||
# A key names one record whatever its type, so a supplied record replaces the arrived one by key alone, and only of the
|
||||
# type that owns it: retyping a key is a removal and an add, so a record another type owns under the key refuses the write.
|
||||
held_by_key = {envelope.key: envelope for envelope in arrived}
|
||||
|
|
@ -314,8 +346,12 @@ def _encode_snapshot(
|
|||
for held in (landing.get(envelope.key), held_by_key.get(envelope.key)):
|
||||
if held is not None and not written_owns(held, envelope):
|
||||
raise retyped(held, envelope)
|
||||
held = held_by_key.get(envelope.key)
|
||||
if held is not None:
|
||||
envelope = replace(envelope, stored_at=stored_at_over(envelope.stored_at, held.stored_at))
|
||||
landing[envelope.key] = envelope
|
||||
supplied = list(landing.values())
|
||||
stamps = {envelope.key: envelope.stored_at for envelope in supplied}
|
||||
replaced = set(landing)
|
||||
kept = [envelope for envelope in arrived if envelope.key not in replaced]
|
||||
envelopes = [*kept, *supplied]
|
||||
|
|
@ -330,7 +366,7 @@ def _encode_snapshot(
|
|||
[envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
|
||||
for envelope in sorted(envelopes, key=lambda item: item.key)
|
||||
]
|
||||
return [format_version, catalog, records]
|
||||
return [format_version, catalog, records], stamps
|
||||
|
||||
def commit_store_file(path: Path, data: bytes) -> None:
|
||||
"""Writes `data` to a unique staged file beside the store, fsyncs it, replaces the store with it and, off Windows,
|
||||
|
|
@ -739,6 +775,49 @@ def _schema_id_for(envelope: CultCacheEnvelope) -> str:
|
|||
return envelope.schema_id or envelope.type
|
||||
|
||||
|
||||
def _parse_stored_at(text: Any) -> datetime | None:
|
||||
"""The instant a stored_at text names, or None: a trailing Z is UTC, fraction digits past the sixth are cut (C# writes
|
||||
seven), a text with no offset is read as UTC. Written to read on Python 3.10, whose fromisoformat takes neither."""
|
||||
if not isinstance(text, str):
|
||||
return None
|
||||
body = text.strip()
|
||||
if body.endswith(("Z", "z")):
|
||||
body = body[:-1] + "+00:00"
|
||||
body = re.sub(r"(\.\d{6})\d+", r"\1", body)
|
||||
try:
|
||||
parsed = datetime.fromisoformat(body)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def mint_stored_at(previous: str | None, now: datetime) -> str:
|
||||
"""A stored_at later than `previous` (C# MintStoredAt): `now`, or the microsecond after `previous` when `now` is not later.
|
||||
A previous that does not parse leaves `now`."""
|
||||
now = now.astimezone(timezone.utc)
|
||||
last = _parse_stored_at(previous)
|
||||
if last is not None and now <= last:
|
||||
now = last + timedelta(microseconds=1)
|
||||
return now.isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
def stored_at_over(staged: str, replaced: str) -> str:
|
||||
"""The stored_at a record replacing `replaced` is written with (C# StoredAtOver): `staged` when both parse and it is strictly
|
||||
later, else one minted over `replaced`. No content is compared: every replacing write is stamped later."""
|
||||
later, earlier = _parse_stored_at(staged), _parse_stored_at(replaced)
|
||||
if later is not None and earlier is not None and later > earlier:
|
||||
return staged
|
||||
return mint_stored_at(replaced, datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def record_version(envelope: CultCacheEnvelope) -> str:
|
||||
"""The content version of a stored record: the lowercase hex SHA-256 of its persisted encoding, the array
|
||||
[key, schema id, stored_at, payload] exactly as the single-file store writes it
|
||||
(variants:ruling:write-set-content-version-declared-ownership)."""
|
||||
record = [envelope.key, _schema_id_for(envelope), envelope.stored_at, envelope.payload]
|
||||
return hashlib.sha256(msgpack.packb(record, use_bin_type=True)).hexdigest()
|
||||
|
||||
|
||||
def _default_catalog_entry(envelope: CultCacheEnvelope) -> CultCacheSchemaCatalogEntry:
|
||||
schema_id = _schema_id_for(envelope)
|
||||
return CultCacheSchemaCatalogEntry(
|
||||
|
|
|
|||
38
packages/cultcache-py/tests/test_content_version_vector.py
Normal file
38
packages/cultcache-py/tests/test_content_version_vector.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import msgpack
|
||||
|
||||
from cultcache_py import SingleFileMessagePackBackingStore
|
||||
from cultcache_py.stores import record_version
|
||||
|
||||
# variants:ruling:write-set-content-version-declared-ownership. The vector in tests/vectors/content-version is written only by
|
||||
# the C# reference; every row of it is computed here from the store file, by the store's read and by record_version.
|
||||
VECTORS = Path(__file__).resolve().parents[3] / "tests" / "vectors" / "content-version"
|
||||
|
||||
|
||||
class ContentVersionVectorTests(unittest.TestCase):
|
||||
def test_every_row_of_the_vector_equals_this_runtimes_version(self) -> None:
|
||||
rows = msgpack.unpackb((VECTORS / "versions.msgpack").read_bytes(), raw=False)
|
||||
self.assertEqual([key for key, _ in rows], ["alpha", "beta"])
|
||||
directory = Path(tempfile.mkdtemp())
|
||||
self.addCleanup(shutil.rmtree, directory, ignore_errors=True)
|
||||
copy = directory / "store.cc"
|
||||
shutil.copyfile(VECTORS / "store.cc", copy)
|
||||
store = SingleFileMessagePackBackingStore(copy)
|
||||
by_key = {envelope.key: envelope for envelope in store.pull_all()}
|
||||
for key, version in rows:
|
||||
with self.subTest(key=key):
|
||||
found = store.read(key)
|
||||
self.assertIsNotNone(found)
|
||||
self.assertEqual(found[1], version)
|
||||
self.assertEqual(record_version(by_key[key]), version)
|
||||
self.assertIsNone(store.read("gamma"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -86,7 +86,7 @@ class CultCacheTests(unittest.TestCase):
|
|||
)
|
||||
cache.pull_all_backing_stores()
|
||||
cache.put(document, "item:potion", Item(name="Potion", category="Consumable", value=50))
|
||||
cache.update(document, "item:potion", lambda item: Item(item.name, item.category, item.value + 5))
|
||||
cache.put(document, "item:potion", Item(name="Potion", category="Consumable", value=55))
|
||||
self.assertEqual(cache.get_required(document, "item:potion").value, 55)
|
||||
cache.delete(document, "item:potion")
|
||||
self.assertIsNone(cache.get(document, "item:potion"))
|
||||
|
|
|
|||
|
|
@ -148,7 +148,6 @@ class UniqueIndexTests(unittest.TestCase):
|
|||
"put": lambda cache, d, k, i: cache.put(d, k, i),
|
||||
"put_envelope": lambda cache, d, k, i: cache.put_envelope(d, envelope(d, k, i)),
|
||||
"put_envelopes": lambda cache, d, k, i: cache.put_envelopes(d, [envelope(d, k, i)]),
|
||||
"update": lambda cache, d, k, i: cache.update(d, k, lambda _held: i),
|
||||
}
|
||||
for name, write in paths.items():
|
||||
with self.subTest(alias=alias, path=name):
|
||||
|
|
|
|||
Loading…
Reference in a new issue