GameCult/CultLib
Watch
9
0
Fork
You've already forked CultLib
 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

C# header: one function decides it for every writer; a store's own ids are the only source

Browse source 
CacheBackingStore.HeaderFor(holdsIds, existingHeader, wholeStore, directoryStore) replaces FormatFor,
DirectoryFormatFor and the writers' private marker logic. A writer that rewrites the whole store
decides by content; a writer that touches part of it keeps a header already marked. Flush,
unconditional and conditional commit, the directory store and CultMesh's single-file write all call
it. A typed CultMesh write replaces the file with a document it can see, so it decides by content;
a raw payload is opaque and keeps a marked file marked.

A record's HoldsIds is what its stored form persists: a load keeps a variant's delta as read, so
the ids a load mints in an override exist only in memory and no longer mark the store. A
conditional commit onto the file no longer reads this cache's view of what the file held, so a
stale view cannot over-mark it. IdPlan.HoldsIds is derived from HeldIds and the fills.

Reverts InternalsVisibleTo GameCult.Caching.MessagePack and the public DirectoryFormatV4/V5
constants: the directory store reaches the decision through the protected member it inherits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-09-29 21:54:01 +02:00
commit 41d0dd9bfb
7 changed files with 164 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/GameCult.Caching.MessagePack/DirectoryMessagePackBackingStore.cs
View file

@ -15,8 +15,9 @@ namespace GameCult.Caching.MessagePack;
// A small hot manifest indexes the store; each record lives in one cold content-addressed page.
public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
{
private const string IndexedFormatVersion = CultPersistedStoreSnapshot.DirectoryFormatV4;
private const string IndexedFormatVersionWithIds = CultPersistedStoreSnapshot.DirectoryFormatV5;
// The two manifest headers; the shared header decision names them.
private static readonly string IndexedFormatVersion = HeaderFor(holdsIds: false, existingHeader: null, wholeStore: true, directoryStore: true);
private static readonly string IndexedFormatVersionWithIds = HeaderFor(holdsIds: true, existingHeader: null, wholeStore: true, directoryStore: true);
// An unleased load that has not settled after this many attempts throws.
private const int UnleasedLoadAttempts = 5;
private readonly FileInfo _manifestFile;
@ -284,12 +285,15 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
pagePayload);
}
// A manifest already marked stays marked: pages it names may hold ids this cache cannot read.
var carriesIds = string.Equals(currentManifest.FormatVersion, IndexedFormatVersionWithIds, StringComparison.Ordinal) ||
keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds);
// The directory store writes only its dirty pages, so it keeps a manifest already marked.
var header = HeaderFor(
keysToWrite.Any(key => Entries.TryGetValue(key, out var written) && written.HoldsIds),
currentManifest.FormatVersion,
wholeStore: false,
directoryStore: true);
WriteManifest(targetCatalog, currentIndex.Values
.OrderBy(record => record.Key, StringComparer.Ordinal)
.ToArray(), carriesIds);
.ToArray(), header);
DeleteUnreferencedRecordPages(currentIndex.Values);
@ -299,11 +303,11 @@ public sealed class DirectoryMessagePackBackingStore : CacheBackingStore
MarkFlushSucceeded();
}
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, bool carriesElementIds)
private void WriteManifest(CultSchemaCatalogEntry[] catalog, CultPersistedRecord[] index, string header)
{
var manifest = new CultPersistedStoreSnapshot
{
FormatVersion = CultPersistedStoreSnapshot.DirectoryFormatFor(carriesElementIds),
FormatVersion = header,
SchemaCatalog = catalog,
Records = index
};

52
src/GameCult.Caching/CultCache.cs
View file

@ -65,21 +65,9 @@ namespace GameCult.Caching
public const string FormatV1 = "cultcache.store.v1";
public const string FormatV2 = "cultcache.store.v2";
public const string FormatV3 = "cultcache.store.v3";
// The directory store's manifest formats: v5 when the directory holds an element id, for the reason v3 exists.
public const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages";
public const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages";
public string FormatVersion { get; set; } = FormatV1;
// The one header decision. v3 when the store holds an element id: a reader older than ids refuses the header, because it
// would skip the id slots and rewrite the elements without them. Otherwise v2 exactly when it holds a variant, and a store
// with neither stays byte-identical v1. A file already marked stays marked; the writer that knows that says so.
internal static string FormatFor(bool holdsIds, bool holdsVariants) =>
holdsIds ? FormatV3 : holdsVariants ? FormatV2 : FormatV1;
// The directory store's header decision (it refuses variants): the same owner as FormatFor.
internal static string DirectoryFormatFor(bool holdsIds) => holdsIds ? DirectoryFormatV5 : DirectoryFormatV4;
public CultSchemaCatalogEntry[] SchemaCatalog { get; set; } = Array.Empty<CultSchemaCatalogEntry>();
public CultPersistedRecord[] Records { get; set; } = Array.Empty<CultPersistedRecord>();
}
@ -2715,7 +2703,9 @@ namespace GameCult.Caching
// decoded copy, and the delta this hold lands carries them.
var bytes = entry.Value;
var idPlan = CultElementIds.Plan(value, key + "." + slot.ToString(CultureInfo.InvariantCulture), deterministic: source != null);
holdsIds |= idPlan.HoldsIds;
// What the record persists: a write persists the ids it mints, a load keeps the delta as the store held it, so
// the ids a load mints exist only in memory.
holdsIds |= source == null ? idPlan.HoldsIds : idPlan.HeldIds;
if (idPlan.Count > 0)
{
idPlan.Apply();
@ -3042,6 +3032,23 @@ namespace GameCult.Caching
private readonly object _detachedGate = new();
internal CultCache? Cache;
private const string DirectoryFormatV4 = "cultcache.store.v4.directory-content-addressed-pages";
private const string DirectoryFormatV5 = "cultcache.store.v5.directory-content-addressed-pages";
// The one header decision, for every writer of every store. The marked header (v3, or the directory store's v5) says the
// store holds an element id: a reader older than ids refuses it, because it would skip the id slots and rewrite the
// elements without them. A writer that rewrites the whole store decides by content, so a store may drop back when nothing
// holds an id. A writer that touches only part of the store keeps a header already marked, because it cannot see what
// the rest holds. holdsIds is about the records this write knows are on disk. Unmarked, a single-file store is v2 exactly
// when it holds a variant, and otherwise v1; the directory store refuses variants and is v4.
protected internal static string HeaderFor(bool holdsIds, string? existingHeader, bool wholeStore, bool directoryStore, bool holdsVariants = false)
{
var marked = directoryStore ? DirectoryFormatV5 : CultPersistedStoreSnapshot.FormatV3;
if (holdsIds || (!wholeStore && string.Equals(existingHeader, marked, StringComparison.Ordinal)))
return marked;
return directoryStore ? DirectoryFormatV4 : holdsVariants ? CultPersistedStoreSnapshot.FormatV2 : CultPersistedStoreSnapshot.FormatV1;
}
// Once attached, a hold on the cache's gate: the store and its cache share one lock, so a load calling back into
// the cache can never take the gate after the store lock, and a direct call publishes what it loaded when it
// returns. File locks are always taken inside it.
@ -3249,7 +3256,9 @@ namespace GameCult.Caching
WriteSnapshot(
Entries.Values.Select(entry => ToPersistedRecord(entry, SerializePayload)),
Entries.Values.Select(entry => entry.Descriptor.ToCatalogEntry()),
Entries.Values.Any(entry => entry.HoldsIds));
Entries.Values.Any(entry => entry.HoldsIds),
existingHeader: null,
wholeStore: true);
Cache?.IdsPersisted(Entries.Keys);
}
@ -3292,14 +3301,13 @@ namespace GameCult.Caching
foreach (var entry in request.Upserts)
records[entry.Key.Value] = ToPersistedRecord(entry, SerializePayload);
// One rule for every writer: the file holds ids when a record it will hold does. A record this commit replaces or removes
// no longer counts. Onto the file, only ids already persisted count (a load-minted id is on no disk), and a file already
// marked stays marked: it may hold records this cache cannot read, and a rewrite must not shed the marker.
// Onto the file, only the batch's records are seen: the rest stay as they are, and the header they carry stays. An
// unconditional commit writes this store's whole view, so what its records hold decides. A record this commit
// replaces or removes no longer counts.
var replaced = request.Upserts.Concat(request.Deletes).Select(entry => entry.Key.Value).ToHashSet(StringComparer.Ordinal);
var holdsIds = request.Upserts.Any(entry => entry.HoldsIds) ||
Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value) && !(ontoDisk && entry.IdsInMemoryOnly)) ||
(ontoDisk && string.Equals(disk.FormatVersion, CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal));
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds);
(!ontoDisk && Entries.Values.Any(entry => entry.HoldsIds && !replaced.Contains(entry.Key.Value)));
WriteSnapshot(records.Values, catalog.Concat(request.Upserts.Select(entry => entry.Descriptor.ToCatalogEntry())), holdsIds, disk.FormatVersion, wholeStore: !ontoDisk);
if (!ontoDisk)
Cache?.IdsPersisted(records.Keys);
foreach (var entry in request.Deletes)
@ -3354,13 +3362,13 @@ namespace GameCult.Caching
}
}
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog, bool holdsElementIds)
private void WriteSnapshot(IEnumerable<CultPersistedRecord> records, IEnumerable<CultSchemaCatalogEntry> catalog, bool holdsElementIds, string? existingHeader, bool wholeStore)
{
var ordered = records.OrderBy(record => record.Key, StringComparer.Ordinal).ToArray();
var used = new HashSet<string>(ordered.Select(record => record.SchemaId), StringComparer.Ordinal);
var snapshot = new CultPersistedStoreSnapshot
{
FormatVersion = CultPersistedStoreSnapshot.FormatFor(holdsElementIds, ordered.Any(record => record.Variant != null)),
FormatVersion = HeaderFor(holdsElementIds, existingHeader, wholeStore, directoryStore: false, holdsVariants: ordered.Any(record => record.Variant != null)),
SchemaCatalog = catalog
.GroupBy(entry => entry.SchemaId, StringComparer.Ordinal)
.Select(group => group.First())

11
src/GameCult.Caching/CultElementIds.cs
View file

@ -64,9 +64,11 @@ namespace GameCult.Caching
public int Count => _fills.Count;
// True when some element under the root holds a non-empty id once the plan is applied: one already there, or one this
// plan fills. It decides whether the store that writes the root carries the v3 header.
public bool HoldsIds { get; internal set; }
// True when some element under the root already holds a non-empty id: one a store persisted, or one the caller set.
public bool HeldIds { get; internal set; }
// True when some element holds a non-empty id once the plan is applied: one already there, or one this plan fills.
public bool HoldsIds => HeldIds || _fills.Count > 0;
internal void Add(object item, Shape shape, string? before, string id) => _fills.Add((item, shape, before, id));
@ -253,7 +255,7 @@ namespace GameCult.Caching
var id = getId(item);
if (string.IsNullOrEmpty(id))
continue;
plan.HoldsIds = true;
plan.HeldIds = true;
if (shape.Derive == null && !IsRandomId(id!))
throw new CultElementIdException(
$"Record {rootPath}: element id '{id}' in the list at {path} is not 12 lowercase hex characters.",
@ -285,7 +287,6 @@ namespace GameCult.Caching
else
id = Mint(itemPath, deterministic, rootPath, taken);
plan.Add(item, itemShape, get(item), id);
plan.HoldsIds = true;
}
Visit(item, itemPath);

1
src/GameCult.Caching/GameCult.Caching.csproj
View file

@ -7,7 +7,6 @@
<ItemGroup>
<InternalsVisibleTo Include="GameCult.Caching.Tests" />
<InternalsVisibleTo Include="GameCult.Mesh" />
<InternalsVisibleTo Include="GameCult.Caching.MessagePack" />
</ItemGroup>
<ItemGroup>

24
src/GameCult.Mesh/CultMeshSingleFileDocuments.cs
View file

@ -110,7 +110,8 @@ namespace GameCult.Mesh
descriptor.ToCatalogEntry(),
storedAt,
CultDocumentMessagePackSerialization.SerializeUntyped(document, typeof(TDocument)),
ids.HoldsIds);
ids.HoldsIds,
contentKnown: true);
}
catch
{
@ -148,7 +149,7 @@ namespace GameCult.Mesh
byte[] payload)
{
if (schema == null) throw new ArgumentNullException(nameof(schema));
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false);
WriteSingleFileDocumentPayload(path, key, schema.ToCatalogEntry(payload), storedAt, payload, holdsIds: false, contentKnown: false);
}
/// <summary>
@ -231,7 +232,8 @@ namespace GameCult.Mesh
CultSchemaCatalogEntry catalogEntry,
string? storedAt,
byte[] payload,
bool holdsIds)
bool holdsIds,
bool contentKnown)
{
if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Value must be non-empty.", nameof(path));
if (catalogEntry == null) throw new ArgumentNullException(nameof(catalogEntry));
@ -239,11 +241,11 @@ namespace GameCult.Mesh
throw new ArgumentException("Catalog entry must include a schema id.", nameof(catalogEntry));
payload ??= Array.Empty<byte>();
// A raw payload is opaque: whether it holds element ids is the typed caller's to say. A file already marked stays
// marked, as it does in a cache's own store.
// A typed write sees the document it replaces the file with, so its ids decide. A raw payload is opaque: the writer
// cannot see whether it holds ids, so a file already marked stays marked.
var snapshot = new CultPersistedStoreSnapshot
{
FormatVersion = CultPersistedStoreSnapshot.FormatFor(holdsIds || IsMarkedOnDisk(path), holdsVariants: false),
FormatVersion = CacheBackingStore.HeaderFor(holdsIds, contentKnown ? null : ExistingHeader(path), wholeStore: contentKnown, directoryStore: false),
SchemaCatalog = new[] { catalogEntry },
Records = new[]
{
@ -264,17 +266,15 @@ namespace GameCult.Mesh
WriteFileAtomically(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
}
private static bool IsMarkedOnDisk(string path)
private static string? ExistingHeader(string path)
{
try
{
var reader = new MessagePackReader(File.ReadAllBytes(path));
reader.ReadArrayHeader();
return string.Equals(reader.ReadString(), CultPersistedStoreSnapshot.FormatV3, StringComparison.Ordinal);
return CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path)).FormatVersion;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or MessagePackSerializationException or EndOfStreamException or InvalidOperationException)
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or MessagePackSerializationException or EndOfStreamException or InvalidOperationException or NotSupportedException)
{
return false;
return null;
}
}

104
tests/GameCult.Caching.Tests/ElementIdTests.cs
View file

@ -810,29 +810,12 @@ namespace GameCult.Caching.Tests
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v3"), "the file holds ids, so it is marked");
}
// A conditional commit lands onto the file as it is, and decides its header as every writer does: by the ids the file
// will hold. Ids on disk mark it whatever header the file carried; ids only this cache minted are on no disk.
// A conditional commit lands onto the file as it is: it sees only its own batch, so ids only this cache minted (they are on no
// disk) do not mark the file.
[Test]
public void AConditionalCommitOntoAFileHoldingIdsMarksItAndOneOntoLoadMintedIdsDoesNot()
public void AConditionalCommitOntoLoadMintedIdsDoesNotMarkTheFile()
{
var held = PathOf("conditional-held.cc");
Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) };
using (var cache = OpenWith(held, false, types))
cache.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d")));
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(held));
snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV1;
File.WriteAllBytes(held, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
using (var cache = OpenWith(held, false, types))
{
Assert.That(cache.Commit(batch =>
{
batch.Expect(new CultRecordKey("plain"), null);
batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain"));
}), Is.True);
}
Assert.That(HeaderOf(held), Is.EqualTo("cultcache.store.v3"), "the deck on disk holds ids");
var preId = PathOf("conditional-minted.cc");
WritePreIdStore(preId, null, ("a", "a", 1));
using (var cache = OpenWith(preId, false, types))
@ -871,6 +854,87 @@ namespace GameCult.Caching.Tests
Assert.That(HeaderOf(replaced), Is.EqualTo("cultcache.store.v1"));
}
// Soul's probe: a C1 v2 store whose variant override holds no ids. Loading mints them in memory; the store keeps the delta
// it read, so the file holds none and no write of a plain record beside it may mark the file.
private static void WriteV2StoreWithAnIdLessVariant(string path)
{
using (var seed = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem)))
{
seed.Commit(batch =>
{
batch.Upsert(typeof(IdDeck), EmptyDeck("base"), new CultRecordKey("base"));
batch.UpsertVariant(new CultRecordKey("v"), new CultRecordKey("base"),
new[] { seed.Override<IdDeck>(nameof(IdDeck.Name), "v"), seed.Override<IdDeck>(nameof(IdDeck.Reels), new List<IdReel> { new() }) });
});
}
var snapshot = CultDocumentMessagePackSerialization.DeserializeSnapshot(File.ReadAllBytes(path));
var record = snapshot.Records.Single(entry => entry.Key == "v");
record.Variant = new CultVariantDelta(record.Variant!.BaseKey, record.Variant.Overrides
.Select(entry => entry.Path[0].Slot == 1 ? CultVariantOverride.Set(1, OldShapedReels("old")) : entry).ToArray());
snapshot.FormatVersion = CultPersistedStoreSnapshot.FormatV2;
File.WriteAllBytes(path, CultDocumentMessagePackSerialization.SerializeSnapshot(snapshot));
}
[TestCase("unconditional")]
[TestCase("conditional")]
[TestCase("flush")]
public void AVariantWhoseIdsExistOnlyInMemoryDoesNotMarkTheFileAPlainWriteLandsIn(string write)
{
var path = PathOf("variant-in-memory-" + write + ".cc");
WriteV2StoreWithAnIdLessVariant(path);
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"));
using (var cache = OpenWith(path, false, typeof(IdDeck), typeof(PreCut2FixtureItem)))
{
Assert.That(AllIds(cache.Get<IdDeck>(new CultRecordKey("v"))!), Has.All.Not.Empty, "the variant holds minted ids in memory");
var key = new CultRecordKey("p");
switch (write)
{
case "unconditional":
UpsertPlain(cache, "p");
break;
case "conditional":
Assert.That(cache.Commit(batch =>
{
batch.Expect(key, null);
batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, key);
}), Is.True);
break;
default:
cache.UpsertAsync(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "p" }, key).GetAwaiter().GetResult();
cache.FlushAllBackingStores();
break;
}
}
Assert.That(DiskRecord(path, "p"), Is.Not.Null);
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v2"), "the file holds no id, whatever this cache minted in memory");
}
// Soul's A/B probe: a conditional commit lands onto the file as it is, so a stale view of what the file once held does not
// decide the header. A holds a deck with ids; B removes it; A commits a plain record conditionally.
[Test]
public void AConditionalCommitIsNotMarkedByIdsItsStaleViewHoldsAndTheFileNoLongerDoes()
{
var path = PathOf("stale-view.cc");
Type[] types = { typeof(IdDeck), typeof(PreCut2FixtureItem) };
using var a = OpenWith(path, false, types);
a.Commit(batch => batch.Upsert(typeof(IdDeck), Deck("d"), new CultRecordKey("d")));
using (var b = OpenWith(path, false, types))
b.Commit(batch => batch.Remove(new CultRecordKey("d")));
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "B's whole-store commit left no id");
Assert.That(a.Get<IdDeck>(new CultRecordKey("d")), Is.Not.Null, "A has not pulled: its view is stale");
Assert.That(a.Commit(batch =>
{
batch.Expect(new CultRecordKey("plain"), null);
batch.Upsert(typeof(PreCut2FixtureItem), new PreCut2FixtureItem { Name = "plain" }, new CultRecordKey("plain"));
}), Is.True);
Assert.That(HeaderOf(path), Is.EqualTo("cultcache.store.v1"), "the merged file holds no id");
}
// Any override holding an id marks the variant, wherever it sits among the overrides.
[Test]
public void AVariantWhoseIdBearingOverrideIsNotTheLastIsStillMarked()

22
tests/GameCult.Mesh.Tests/CultMeshSingleFileElementIdTests.cs
View file

@ -62,8 +62,9 @@ public sealed class CultMeshSingleFileElementIdTests
.Should().Equal(publication.Representations.Select(entry => entry.Id));
}
// A typed write replaces the whole file with a document it can see, so the ids that document holds decide the header.
[Test]
public void TypedWriteOfADocumentHoldingNoIdStaysV1AndAMarkedFileStaysMarked()
public void TypedWriteDecidesTheHeaderByTheDocumentItWritesEvenOverAMarkedFile()
{
var path = Path_("none.cc");
CultMesh.WriteSingleFileDocument(path, Key, Publication());
@ -72,7 +73,24 @@ public sealed class CultMeshSingleFileElementIdTests
CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor()));
HeaderOf(path).Should().Be("cultcache.store.v3");
CultMesh.WriteSingleFileDocument(path, Key, Publication());
HeaderOf(path).Should().Be("cultcache.store.v3", "a file already marked on disk stays marked");
HeaderOf(path).Should().Be("cultcache.store.v1", "the whole file is replaced by a document that holds no id");
}
// A raw payload is opaque: the writer cannot see what it holds, so a marked file stays marked.
[Test]
public void RawPayloadWriteOverAMarkedFileKeepsItMarkedAndOverAnUnmarkedOneStaysV1()
{
var path = Path_("raw.cc");
var schema = new CultMeshSingleFileDocumentSchema("raw:schema", "RawSchema", "1");
var payload = new byte[] { 0x90 };
CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, payload);
HeaderOf(path).Should().Be("cultcache.store.v1");
CultMesh.WriteSingleFileDocument(path, Key, Publication(Descriptor()));
HeaderOf(path).Should().Be("cultcache.store.v3");
CultMesh.WriteSingleFileDocumentPayload(path, Key, schema, null, payload);
HeaderOf(path).Should().Be("cultcache.store.v3", "the payload's ids are not the writer's to see");
}
[Test]