C# header: one function decides it for every writer; a store's own ids are the only source
CacheBackingStore.HeaderFor(holdsIds, existingHeader, wholeStore, directoryStore) replaces FormatFor, DirectoryFormatFor and the writers' private marker logic. A writer that rewrites the whole store decides by content; a writer that touches part of it keeps a header already marked. Flush, unconditional and conditional commit, the directory store and CultMesh's single-file write all call it. A typed CultMesh write replaces the file with a document it can see, so it decides by content; a raw payload is opaque and keeps a marked file marked. A record's HoldsIds is what its stored form persists: a load keeps a variant's delta as read, so the ids a load mints in an override exist only in memory and no longer mark the store. A conditional commit onto the file no longer reads this cache's view of what the file held, so a stale view cannot over-mark it. IdPlan.HoldsIds is derived from HeldIds and the fills. Reverts InternalsVisibleTo GameCult.Caching.MessagePack and the public DirectoryFormatV4/V5 constants: the directory store reaches the decision through the protected member it inherits. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
591eeff70d
commit
41d0dd9bfb
7 changed files with 164 additions and 70 deletions
|
|
@ -15,8 +15,9 @@ namespace GameCult.Caching.MessagePack;
|
|||
// A small hot manifest indexes the store; each record lives in one cold content-addressed page.
|
||||
public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
||||
{
|
||||
private const string IndexedFormatVersion = CultPersistedStoreSnapshot.DirectoryFormatV4;
|
||||
private const string IndexedFormatVersionWithIds = CultPersistedStoreSnapshot.DirectoryFormatV5;
|
||||
// The two manifest headers; the shared header decision names them.
|
||||
private static readonly string IndexedFormatVersion = HeaderFor(holdsIds: false, existingHeader: null, wholeStore: true, directoryStore: true);
|
||||
private static readonly string IndexedFormatVersionWithIds = HeaderFor(holdsIds: true, existingHeader: null, wholeStore: true, directoryStore: true);
|
||||
// An unleased load that has not settled after this many attempts throws.
|
||||
private const int UnleasedLoadAttempts = 5;
|
||||
private readonly FileInfo _manifestFile;
|
||||
|
|
@ -284,12 +285,15 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
|||
pagePayload);
|
||||
}
|
||||
|
||||
// A manifest already marked stays marked: pages it names may hold ids this cache cannot read.
|
||||
var carriesIds = string.Equals(currentManifest.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal) ||
|
||||
keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds);
|
||||
// The directory store writes only its dirty pages, so it keeps a manifest already marked.
|
||||
var header = HeaderFor(
|
||||
keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds),
|
||||
currentManifest.FormatVersion,
|
||||
wholeStore: false,
|
||||
directoryStore: true);
|
||||
WriteManifest(targetCatalog, currentIndex.Values
|
||||
.OrderBy(record => record.Key, StringComparer.Ordinal)
|
||||
.ToArray(), carriesIds);
|
||||
.ToArray(), header);
|
||||
|
||||
DeleteUnreferencedRecordPages(currentIndex.Values);
|
||||
|
||||
|
|
@ -299,11 +303,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
|
|||
MarkFlushSucceeded();
|
||||
}
|
||||
|
||||
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, bool carriesElementIds)
|
||||
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, string header)
|
||||
{
|
||||
var manifest = new CultPersistedStoreSnapshot
|
||||
{
|
||||
FormatVersion = CultPersistedStoreSnapshot.DirectoryFormatFor(carriesElementIds),
|
||||
FormatVersion = header,
|
||||
SchemaCatalog = catalog,
|
||||
Records = index
|
||||
};
|
||||
|
|
|
|||
|
|
@ -65,21 +65,9 @@ namespace GameCult.Caching
|
|||
public const string FormatV1 = "cultcache.store.v1";
|
||||
public const string FormatV2 = "cultcache.store.v2";
|
||||
public const string FormatV3 = "cultcache.store.v3";
|
||||
// The directory store's manifest formats: v5 when the directory holds an element id, for the reason v3 exists.
|
||||
public const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages";
|
||||
public const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages";
|
||||
|
||||
public string FormatVersion { get; set; } = FormatV1;
|
||||
|
||||
// The one header decision. v3 when the store holds an element id: a reader older than ids refuses the header, because it
|
||||
// would skip the id slots and rewrite the elements without them. Otherwise v2 exactly when it holds a variant, and a store
|
||||
// with neither stays byte-identical v1. A file already marked stays marked; the writer that knows that says so.
|
||||
internal static string FormatFor(bool holdsIds, bool holdsVariants) =>
|
||||
holdsIds ? FormatV3 : holdsVariants ? FormatV2 : FormatV1;
|
||||
|
||||
// The directory store's header decision (it refuses variants): the same owner as FormatFor.
|
||||
internal static string DirectoryFormatFor(bool holdsIds) => holdsIds ? DirectoryFormatV5 : DirectoryFormatV4;
|
||||
|
||||
public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty<CultSchemaCatalogEntry>();
|
||||
public CultPersistedRecord[] Records { get; set; } = Array.Empty<CultPersistedRecord>();
|
||||
}
|
||||
|
|
@ -2715,7 +2703,9 @@ namespace GameCult.Caching
|
|||
// decoded copy, and the delta this hold lands carries them.
|
||||
var bytes = entry.Value;
|
||||
var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null);
|
||||
holdsIds |= idPlan.HoldsIds;
|
||||
// What the record persists: a write persists the ids it mints, a load keeps the delta as the store held it, so
|
||||
// the ids a load mints exist only in memory.
|
||||
holdsIds |= source == null ? idPlan.HoldsIds : idPlan.HeldIds;
|
||||
if (idPlan.Count > 0)
|
||||
{
|
||||
idPlan.Apply();
|
||||
|
|
@ -3042,6 +3032,23 @@ namespace GameCult.Caching
|
|||
private readonly object _detachedGate = new();
|
||||
internal CultCache? Cache;
|
||||
|
||||
private const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages";
|
||||
private const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages";
|
||||
|
||||
// The one header decision, for every writer of every store. The marked header (v3, or the directory store's v5) says the
|
||||
// store holds an element id: a reader older than ids refuses it, because it would skip the id slots and rewrite the
|
||||
// elements without them. A writer that rewrites the whole store decides by content, so a store may drop back when nothing
|
||||
// holds an id. A writer that touches only part of the store keeps a header already marked, because it cannot see what
|
||||
// the rest holds. holdsIds is about the records this write knows are on disk. Unmarked, a single-file store is v2 exactly
|
||||
// when it holds a variant, and otherwise v1; the directory store refuses variants and is v4.
|
||||
protected internal static string HeaderFor(bool holdsIds, string? existingHeader, bool wholeStore, bool directoryStore, bool holdsVariants = false)
|
||||
{
|
||||
var marked = directoryStore ? DirectoryFormatV5 : CultPersistedStoreSnapshot.FormatV3;
|
||||
if (holdsIds || (!wholeStore && string.Equals(existingHeader, marked, StringComparison.Ordinal)))
|
||||
return marked;
|
||||
return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1;
|
||||
}
|
||||
|
||||
// Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into
|
||||
// the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it
|
||||
// returns. File locks are always taken inside it.
|
||||
|
|
@ -3249,7 +3256,9 @@ namespace GameCult.Caching
|
|||
WriteSnapshot(
|
||||
Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)),
|
||||
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()),
|
||||
Entries.Values.Any(entry => entry.HoldsIds));
|
||||
Entries.Values.Any(entry => entry.HoldsIds),
|
||||
existingHeader: null,
|
||||
wholeStore: true);
|
||||
Cache?.IdsPersisted(Entries.Keys);
|
||||
}
|
||||
|
||||
|
|
@ -3292,14 +3301,13 @@ namespace GameCult.Caching
|
|||
foreach (var entry in request.Upserts)
|
||||
records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload);
|
||||
|
||||
// One rule for every writer: the file holds ids when a record it will hold does. A record this commit replaces or removes
|
||||
// no longer counts. Onto the file, only ids already persisted count (a load-minted id is on no disk), and a file already
|
||||
// marked stays marked: it may hold records this cache cannot read, and a rewrite must not shed the marker.
|
||||
// Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An
|
||||
// unconditional commit writes this store's whole view, so what its records hold decides. A record this commit
|
||||
// replaces or removes no longer counts.
|
||||
var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal);
|
||||
var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) ||
|
||||
Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value) && !(ontoDisk && entry.IdsInMemoryOnly)) ||
|
||||
(ontoDisk && string.Equals(disk.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal));
|
||||
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds);
|
||||
(!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value)));
|
||||
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk);
|
||||
if (!ontoDisk)
|
||||
Cache?.IdsPersisted(records.Keys);
|
||||
foreach (var entry in request.Deletes)
|
||||
|
|
@ -3354,13 +3362,13 @@ namespace GameCult.Caching
|
|||
}
|
||||
}
|
||||
|
||||
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog, bool holdsElementIds)
|
||||
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog, bool holdsElementIds, string? existingHeader, bool wholeStore)
|
||||
{
|
||||
var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray();
|
||||
var used = new HashSet<string>(ordered.Select(record => record.SchemaId), StringComparer.Ordinal);
|
||||
var snapshot = new CultPersistedStoreSnapshot
|
||||
{
|
||||
FormatVersion = CultPersistedStoreSnapshot.FormatFor(holdsElementIds, ordered.Any(record => record.Variant != null)),
|
||||
FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)),
|
||||
SchemaCatalog = catalog
|
||||
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
|
||||
.Select(group => group.First())
|
||||
|
|
|
|||
|
|
@ -64,9 +64,11 @@ namespace GameCult.Caching
|
|||
|
||||
public int Count => _fills.Count;
|
||||
|
||||
// True when some element under the root holds a non-empty id once the plan is applied: one already there, or one this
|
||||
// plan fills. It decides whether the store that writes the root carries the v3 header.
|
||||
public bool HoldsIds { get; internal set; }
|
||||
// True when some element under the root already holds a non-empty id: one a store persisted, or one the caller set.
|
||||
public bool HeldIds { get; internal set; }
|
||||
|
||||
// True when some element holds a non-empty id once the plan is applied: one already there, or one this plan fills.
|
||||
public bool HoldsIds => HeldIds || _fills.Count > 0;
|
||||
|
||||
internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id));
|
||||
|
||||
|
|
@ -253,7 +255,7 @@ namespace GameCult.Caching
|
|||
var id = getId(item);
|
||||
if (string.IsNullOrEmpty(id))
|
||||
continue;
|
||||
plan.HoldsIds = true;
|
||||
plan.HeldIds = true;
|
||||
if (shape.Derive == null && !IsRandomId(id!))
|
||||
throw new CultElementIdException(
|
||||
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters.",
|
||||
|
|
@ -285,7 +287,6 @@ namespace GameCult.Caching
|
|||
else
|
||||
id = Mint(itemPath, deterministic, rootPath, taken);
|
||||
plan.Add(item, itemShape, get(item), id);
|
||||
plan.HoldsIds = true;
|
||||
}
|
||||
|
||||
Visit(item, itemPath);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
|
||||
<InternalsVisibleTo Include="GameCult.Mesh" />
|
||||
<InternalsVisibleTo Include="GameCult.Caching.MessagePack" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -110,7 +110,8 @@ namespace GameCult.Mesh
|
|||
descriptor.ToCatalogEntry(),
|
||||
storedAt,
|
||||
CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)),
|
||||
ids.HoldsIds);
|
||||
ids.HoldsIds,
|
||||
contentKnown: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
|
|
@ -148,7 +149,7 @@ namespace GameCult.Mesh
|
|||
byte[] payload)
|
||||
{
|
||||
if (schema == null) throw new ArgumentNullException(nameof(schema));
|
||||
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false);
|
||||
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false, contentKnown: false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -231,7 +232,8 @@ namespace GameCult.Mesh
|
|||
CultSchemaCatalogEntry catalogEntry,
|
||||
string? storedAt,
|
||||
byte[] payload,
|
||||
bool holdsIds)
|
||||
bool holdsIds,
|
||||
bool contentKnown)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
|
||||
if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry));
|
||||
|
|
@ -239,11 +241,11 @@ namespace GameCult.Mesh
|
|||
throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry));
|
||||
|
||||
payload ??= Array.Empty<byte>();
|
||||
// A raw payload is opaque: whether it holds element ids is the typed caller's to say. A file already marked stays
|
||||
// marked, as it does in a cache's own store.
|
||||
// A typed write sees the document it replaces the file with, so its ids decide. A raw payload is opaque: the writer
|
||||
// cannot see whether it holds ids, so a file already marked stays marked.
|
||||
var snapshot = new CultPersistedStoreSnapshot
|
||||
{
|
||||
FormatVersion = CultPersistedStoreSnapshot.FormatFor(holdsIds || IsMarkedOnDisk(path), holdsVariants: false),
|
||||
FormatVersion = CacheBackingStore.HeaderFor(holdsIds, contentKnown ? null : ExistingHeader(path), wholeStore: contentKnown, directoryStore: false),
|
||||
SchemaCatalog = new[] { catalogEntry },
|
||||
Records = new[]
|
||||
{
|
||||
|
|
@ -264,17 +266,15 @@ namespace GameCult.Mesh
|
|||
WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
|
||||
}
|
||||
|
||||
private static bool IsMarkedOnDisk(string path)
|
||||
private static string? ExistingHeader(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reader = new MessagePackReader(File.ReadAllBytes(path));
|
||||
reader.ReadArrayHeader();
|
||||
return string.Equals(reader.ReadString(), CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal);
|
||||
return CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or MessagePackSerializationException or EndOfStreamException or InvalidOperationException)
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or MessagePackSerializationException or EndOfStreamException or InvalidOperationException or NotSupportedException)
|
||||
{
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue